pytaskq 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq/__init__.py +6 -0
- pytaskq/__main__.py +4 -0
- pytaskq/api/router.py +15 -0
- pytaskq/api/v1/auth.py +235 -0
- pytaskq/api/v1/dashboard.py +99 -0
- pytaskq/api/v1/dlq.py +114 -0
- pytaskq/api/v1/system.py +68 -0
- pytaskq/api/v1/tasks.py +99 -0
- pytaskq/app.py +324 -0
- pytaskq/cli.py +250 -0
- pytaskq/core/auth.py +535 -0
- pytaskq/core/config.py +52 -0
- pytaskq/core/dependencies.py +106 -0
- pytaskq/core/exceptions.py +12 -0
- pytaskq/core/idempotency.py +45 -0
- pytaskq/core/queueing.py +282 -0
- pytaskq/core/redis_client.py +19 -0
- pytaskq/main.py +124 -0
- pytaskq/net.py +67 -0
- pytaskq/schemas/requests.py +32 -0
- pytaskq/services/dashboard_service.py +42 -0
- pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq/static/favicon.svg +1 -0
- pytaskq/static/index.html +15 -0
- pytaskq/task_registery.py +144 -0
- pytaskq/tracing.py +98 -0
- pytaskq/worker.py +636 -0
- pytaskq-0.3.0.dist-info/METADATA +262 -0
- pytaskq-0.3.0.dist-info/RECORD +34 -0
- pytaskq-0.3.0.dist-info/WHEEL +5 -0
- pytaskq-0.3.0.dist-info/entry_points.txt +2 -0
- pytaskq-0.3.0.dist-info/licenses/LICENSE +21 -0
- pytaskq-0.3.0.dist-info/top_level.txt +1 -0
pytaskq/__init__.py
ADDED
pytaskq/__main__.py
ADDED
pytaskq/api/router.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from fastapi import APIRouter
|
|
2
|
+
from pytaskq.api.v1.tasks import router as tasks_router, list_router as task_list_router
|
|
3
|
+
from pytaskq.api.v1.dlq import router as dlq_router
|
|
4
|
+
from pytaskq.api.v1.dashboard import router as dashboard_router
|
|
5
|
+
from pytaskq.api.v1.system import router as system_router
|
|
6
|
+
from pytaskq.api.v1.auth import router as auth_router
|
|
7
|
+
|
|
8
|
+
api_router = APIRouter()
|
|
9
|
+
|
|
10
|
+
api_router.include_router(tasks_router)
|
|
11
|
+
api_router.include_router(task_list_router)
|
|
12
|
+
api_router.include_router(dlq_router)
|
|
13
|
+
api_router.include_router(dashboard_router)
|
|
14
|
+
api_router.include_router(system_router)
|
|
15
|
+
api_router.include_router(auth_router)
|
pytaskq/api/v1/auth.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""
|
|
2
|
+
/auth — sign-in, first-run setup, invites, users and API tokens.
|
|
3
|
+
|
|
4
|
+
Public: GET /auth/status, POST /auth/setup, POST /auth/login,
|
|
5
|
+
GET /auth/invites/{token}, POST /auth/invites/{token}/accept
|
|
6
|
+
Signed in: GET /auth/me, POST /auth/logout, POST /auth/me/password,
|
|
7
|
+
GET|POST /auth/me/tokens, DELETE /auth/me/tokens/{id}
|
|
8
|
+
Admin: GET /auth/users, PATCH|DELETE /auth/users/{username},
|
|
9
|
+
GET|POST /auth/invites, DELETE /auth/invites/{id}
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from typing import Literal, Optional
|
|
13
|
+
|
|
14
|
+
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
|
15
|
+
from pydantic import BaseModel, Field
|
|
16
|
+
|
|
17
|
+
from pytaskq.core import auth, redis_client
|
|
18
|
+
from pytaskq.core.config import settings
|
|
19
|
+
from pytaskq.core.dependencies import SESSION_COOKIE, current_user, get_client_ip, require
|
|
20
|
+
|
|
21
|
+
router = APIRouter(prefix="/auth", tags=["auth"])
|
|
22
|
+
|
|
23
|
+
Role = Literal["viewer", "operator", "admin"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _fail(e: auth.AuthError):
|
|
27
|
+
raise HTTPException(status_code=e.status, detail=e.message)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _secure_cookie(request: Request) -> bool:
|
|
31
|
+
mode = settings.COOKIE_SECURE.strip().lower()
|
|
32
|
+
if mode in ("true", "1", "yes", "on"):
|
|
33
|
+
return True
|
|
34
|
+
if mode in ("false", "0", "no", "off"):
|
|
35
|
+
return False
|
|
36
|
+
return request.url.scheme == "https" or request.headers.get("x-forwarded-proto", "").lower() == "https"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _set_session(response: Response, request: Request, token: str) -> None:
|
|
40
|
+
response.set_cookie(
|
|
41
|
+
SESSION_COOKIE, token,
|
|
42
|
+
max_age=auth.session_ttl(), httponly=True, samesite="lax",
|
|
43
|
+
secure=_secure_cookie(request), path="/",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def _sign_in(response: Response, request: Request, user: auth.User) -> dict:
|
|
48
|
+
token = await auth.create_session(redis_client.r, user.username)
|
|
49
|
+
_set_session(response, request, token)
|
|
50
|
+
return {"user": user.public()}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ── public ──────────────────────────────────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
class Credentials(BaseModel):
|
|
56
|
+
username: str = Field(..., max_length=64)
|
|
57
|
+
password: str = Field(..., max_length=auth.MAX_PASSWORD)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class SetupRequest(Credentials):
|
|
61
|
+
setup_code: str = Field(..., max_length=64)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@router.get("/status")
|
|
65
|
+
async def status(request: Request):
|
|
66
|
+
"""What the dashboard should show: setup screen, sign-in screen, or the app."""
|
|
67
|
+
r = redis_client.r
|
|
68
|
+
out = {"auth_enabled": settings.auth_enabled, "setup_required": await auth.setup_required(r), "user": None}
|
|
69
|
+
try:
|
|
70
|
+
out["user"] = (await current_user(request)).public()
|
|
71
|
+
except HTTPException:
|
|
72
|
+
pass
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@router.post("/setup")
|
|
77
|
+
async def setup(body: SetupRequest, request: Request, response: Response):
|
|
78
|
+
"""Create the first admin account using the setup code from the server log."""
|
|
79
|
+
try:
|
|
80
|
+
user = await auth.complete_setup(redis_client.r, body.setup_code, body.username, body.password)
|
|
81
|
+
except auth.AuthError as e:
|
|
82
|
+
_fail(e)
|
|
83
|
+
return await _sign_in(response, request, user)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@router.post("/login")
|
|
87
|
+
async def login(body: Credentials, request: Request, response: Response):
|
|
88
|
+
if not settings.auth_enabled:
|
|
89
|
+
raise HTTPException(status_code=400, detail="Sign-in is turned off on this server (PYTASKQ_AUTH=off).")
|
|
90
|
+
try:
|
|
91
|
+
user = await auth.authenticate(redis_client.r, body.username, body.password, ip=get_client_ip(request))
|
|
92
|
+
except auth.AuthError as e:
|
|
93
|
+
_fail(e)
|
|
94
|
+
return await _sign_in(response, request, user)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@router.get("/invites/{token}")
|
|
98
|
+
async def view_invite(token: str):
|
|
99
|
+
"""Lets the invite screen show which role the link grants (public)."""
|
|
100
|
+
invite = await auth.get_invite(redis_client.r, token)
|
|
101
|
+
if not invite:
|
|
102
|
+
raise HTTPException(status_code=404, detail="This invite link is invalid or has expired. Ask an admin for a new one.")
|
|
103
|
+
return {"role": invite["role"], "created_by": invite["created_by"], "expires_at": invite["expires_at"]}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@router.post("/invites/{token}/accept")
|
|
107
|
+
async def accept_invite(token: str, body: Credentials, request: Request, response: Response):
|
|
108
|
+
try:
|
|
109
|
+
user = await auth.accept_invite(redis_client.r, token, body.username, body.password)
|
|
110
|
+
except auth.AuthError as e:
|
|
111
|
+
_fail(e)
|
|
112
|
+
return await _sign_in(response, request, user)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# ── signed in ───────────────────────────────────────────────────────────────────
|
|
116
|
+
|
|
117
|
+
@router.post("/logout")
|
|
118
|
+
async def logout(request: Request, response: Response):
|
|
119
|
+
await auth.end_session(redis_client.r, request.cookies.get(SESSION_COOKIE, ""))
|
|
120
|
+
response.delete_cookie(SESSION_COOKIE, path="/")
|
|
121
|
+
return {"ok": True}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@router.get("/me")
|
|
125
|
+
async def me(user: auth.User = Depends(current_user)):
|
|
126
|
+
return {"user": user.public()}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class PasswordChange(BaseModel):
|
|
130
|
+
current_password: str = Field(..., max_length=auth.MAX_PASSWORD)
|
|
131
|
+
new_password: str = Field(..., max_length=auth.MAX_PASSWORD)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@router.post("/me/password")
|
|
135
|
+
async def change_password(body: PasswordChange, request: Request, user: auth.User = Depends(current_user)):
|
|
136
|
+
if user.via != "session":
|
|
137
|
+
raise HTTPException(status_code=400, detail="Change your password from the dashboard while signed in.")
|
|
138
|
+
try:
|
|
139
|
+
await auth.authenticate(redis_client.r, user.username, body.current_password, ip=get_client_ip(request))
|
|
140
|
+
except auth.AuthError as e:
|
|
141
|
+
if e.status == 401:
|
|
142
|
+
raise HTTPException(status_code=400, detail="Your current password is not right.")
|
|
143
|
+
_fail(e)
|
|
144
|
+
keep = auth.session_sha(request.cookies.get(SESSION_COOKIE, ""))
|
|
145
|
+
try:
|
|
146
|
+
await auth.set_password(redis_client.r, user.username, body.new_password, keep_session_sha=keep)
|
|
147
|
+
except auth.AuthError as e:
|
|
148
|
+
_fail(e)
|
|
149
|
+
return {"ok": True, "message": "Password changed. You were signed out everywhere else."}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _real_user(user: auth.User) -> auth.User:
|
|
153
|
+
if user.via in ("auth_off", "master_key"):
|
|
154
|
+
raise HTTPException(status_code=400, detail="API tokens belong to a user account; sign in as a user first.")
|
|
155
|
+
return user
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class TokenRequest(BaseModel):
|
|
159
|
+
name: str = Field("API token", max_length=60)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@router.get("/me/tokens")
|
|
163
|
+
async def my_tokens(user: auth.User = Depends(current_user)):
|
|
164
|
+
_real_user(user)
|
|
165
|
+
return {"tokens": await auth.list_api_tokens(redis_client.r, user.username)}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@router.post("/me/tokens")
|
|
169
|
+
async def create_token(body: TokenRequest, user: auth.User = Depends(current_user)):
|
|
170
|
+
_real_user(user)
|
|
171
|
+
token, info = await auth.create_api_token(redis_client.r, user.username, body.name)
|
|
172
|
+
return {"token": token, "info": info,
|
|
173
|
+
"message": "Copy this token now — it won't be shown again."}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@router.delete("/me/tokens/{token_id}")
|
|
177
|
+
async def delete_token(token_id: str, user: auth.User = Depends(current_user)):
|
|
178
|
+
_real_user(user)
|
|
179
|
+
if not await auth.revoke_api_token(redis_client.r, user.username, token_id):
|
|
180
|
+
raise HTTPException(status_code=404, detail="Token not found.")
|
|
181
|
+
return {"ok": True}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# ── admin ───────────────────────────────────────────────────────────────────────
|
|
185
|
+
|
|
186
|
+
@router.get("/users")
|
|
187
|
+
async def users(_: auth.User = Depends(require("admin"))):
|
|
188
|
+
return {"users": [u.public() for u in await auth.list_users(redis_client.r)]}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class UserUpdate(BaseModel):
|
|
192
|
+
role: Optional[Role] = None
|
|
193
|
+
disabled: Optional[bool] = None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@router.patch("/users/{username}")
|
|
197
|
+
async def update_user(username: str, body: UserUpdate, admin: auth.User = Depends(require("admin"))):
|
|
198
|
+
try:
|
|
199
|
+
user = await auth.update_user(redis_client.r, username, role=body.role, disabled=body.disabled, acting=admin)
|
|
200
|
+
except auth.AuthError as e:
|
|
201
|
+
_fail(e)
|
|
202
|
+
return {"user": user.public()}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@router.delete("/users/{username}")
|
|
206
|
+
async def remove_user(username: str, admin: auth.User = Depends(require("admin"))):
|
|
207
|
+
if username == admin.username:
|
|
208
|
+
raise HTTPException(status_code=400, detail="You can't delete your own account.")
|
|
209
|
+
try:
|
|
210
|
+
await auth.delete_user(redis_client.r, username)
|
|
211
|
+
except auth.AuthError as e:
|
|
212
|
+
_fail(e)
|
|
213
|
+
return {"ok": True}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class InviteRequest(BaseModel):
|
|
217
|
+
role: Role = "viewer"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@router.get("/invites")
|
|
221
|
+
async def invites(_: auth.User = Depends(require("admin"))):
|
|
222
|
+
return {"invites": await auth.list_invites(redis_client.r)}
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@router.post("/invites")
|
|
226
|
+
async def create_invite(body: InviteRequest, admin: auth.User = Depends(require("admin"))):
|
|
227
|
+
token, info = await auth.create_invite(redis_client.r, body.role, created_by=admin.username)
|
|
228
|
+
return {"token": token, "path": f"/?invite={token}", "invite": info}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@router.delete("/invites/{invite_id}")
|
|
232
|
+
async def revoke_invite(invite_id: str, _: auth.User = Depends(require("admin"))):
|
|
233
|
+
if not await auth.revoke_invite(redis_client.r, invite_id):
|
|
234
|
+
raise HTTPException(status_code=404, detail="Invite not found (it may have been used or expired).")
|
|
235
|
+
return {"ok": True}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
|
|
4
|
+
from pytaskq.core import redis_client
|
|
5
|
+
from pytaskq.core.config import settings
|
|
6
|
+
from pytaskq.core.dependencies import _same_origin, authenticate_request, require
|
|
7
|
+
from pytaskq.services.dashboard_service import _get_dashboard_data
|
|
8
|
+
|
|
9
|
+
router = APIRouter(tags=["dashboard"])
|
|
10
|
+
logger = logging.getLogger("api")
|
|
11
|
+
|
|
12
|
+
@router.get("/metrics", dependencies=[Depends(require("viewer"))])
|
|
13
|
+
async def metrics(req: Request):
|
|
14
|
+
data = await _get_dashboard_data("global")
|
|
15
|
+
return data["metrics"]
|
|
16
|
+
|
|
17
|
+
@router.websocket("/ws/dashboard")
|
|
18
|
+
async def dashboard_websocket(websocket: WebSocket):
|
|
19
|
+
# Browsers send cookies with WebSocket handshakes from any site, so check
|
|
20
|
+
# the Origin as well as the session. Scripts can pass ?token=ptq_… instead.
|
|
21
|
+
if settings.auth_enabled:
|
|
22
|
+
headers = dict(websocket.headers)
|
|
23
|
+
token = websocket.query_params.get("token")
|
|
24
|
+
if token:
|
|
25
|
+
headers["authorization"] = f"Bearer {token}"
|
|
26
|
+
try:
|
|
27
|
+
if not _same_origin(headers.get("origin", ""), headers.get("host", "")):
|
|
28
|
+
raise HTTPException(status_code=403)
|
|
29
|
+
user = await authenticate_request(headers, websocket.cookies, "GET")
|
|
30
|
+
if not user.can("viewer"):
|
|
31
|
+
raise HTTPException(status_code=403)
|
|
32
|
+
except HTTPException:
|
|
33
|
+
await websocket.close(code=4401)
|
|
34
|
+
return
|
|
35
|
+
await websocket.accept()
|
|
36
|
+
logger.info("[WS] Dashboard connected")
|
|
37
|
+
|
|
38
|
+
push_event = asyncio.Event()
|
|
39
|
+
stop_event = asyncio.Event()
|
|
40
|
+
tracked_ids: list[str] = []
|
|
41
|
+
|
|
42
|
+
async def redis_listener():
|
|
43
|
+
pubsub = redis_client.r.pubsub()
|
|
44
|
+
await pubsub.subscribe("events:global")
|
|
45
|
+
try:
|
|
46
|
+
while not stop_event.is_set():
|
|
47
|
+
msg = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
|
|
48
|
+
if msg:
|
|
49
|
+
push_event.set()
|
|
50
|
+
finally:
|
|
51
|
+
await pubsub.unsubscribe("events:global")
|
|
52
|
+
await pubsub.aclose()
|
|
53
|
+
|
|
54
|
+
async def client_listener():
|
|
55
|
+
nonlocal tracked_ids
|
|
56
|
+
try:
|
|
57
|
+
while not stop_event.is_set():
|
|
58
|
+
msg = await websocket.receive_json()
|
|
59
|
+
if "track" in msg and isinstance(msg["track"], list):
|
|
60
|
+
tracked_ids = msg["track"][:10]
|
|
61
|
+
push_event.set()
|
|
62
|
+
except (WebSocketDisconnect, Exception):
|
|
63
|
+
stop_event.set()
|
|
64
|
+
|
|
65
|
+
async def pusher():
|
|
66
|
+
push_event.set()
|
|
67
|
+
while not stop_event.is_set():
|
|
68
|
+
try:
|
|
69
|
+
await asyncio.wait_for(push_event.wait(), timeout=30)
|
|
70
|
+
except asyncio.TimeoutError:
|
|
71
|
+
pass
|
|
72
|
+
push_event.clear()
|
|
73
|
+
|
|
74
|
+
if stop_event.is_set():
|
|
75
|
+
break
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
data = await _get_dashboard_data("global")
|
|
79
|
+
task_statuses = {}
|
|
80
|
+
for tid in tracked_ids:
|
|
81
|
+
res = await redis_client.r.hgetall(f"task:{tid}")
|
|
82
|
+
if res:
|
|
83
|
+
task_statuses[tid] = res.get("status", "Unknown")
|
|
84
|
+
data["task_statuses"] = task_statuses
|
|
85
|
+
await websocket.send_json(data)
|
|
86
|
+
except Exception:
|
|
87
|
+
stop_event.set()
|
|
88
|
+
break
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
await asyncio.gather(
|
|
92
|
+
redis_listener(),
|
|
93
|
+
client_listener(),
|
|
94
|
+
pusher(),
|
|
95
|
+
return_exceptions=True,
|
|
96
|
+
)
|
|
97
|
+
finally:
|
|
98
|
+
stop_event.set()
|
|
99
|
+
logger.info("[WS] Dashboard disconnected")
|
pytaskq/api/v1/dlq.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
from fastapi import APIRouter, Depends, HTTPException
|
|
5
|
+
|
|
6
|
+
from pytaskq.core import queueing, redis_client
|
|
7
|
+
from pytaskq.core.dependencies import require
|
|
8
|
+
from pytaskq.services.dashboard_service import _notify_tenant
|
|
9
|
+
|
|
10
|
+
router = APIRouter(prefix="/dlq", tags=["dlq"], dependencies=[Depends(require("viewer"))])
|
|
11
|
+
_operator = [Depends(require("operator"))]
|
|
12
|
+
|
|
13
|
+
DLQ_KEY = "dlq"
|
|
14
|
+
_DLQ_ONLY_FIELDS = ("error", "failed_at")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@router.get("")
|
|
18
|
+
async def get_dlq():
|
|
19
|
+
view_dlq = await redis_client.r.lrange(DLQ_KEY, 0, -1)
|
|
20
|
+
tasks = [json.loads(item) for item in view_dlq]
|
|
21
|
+
return {"tasks": tasks}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
async def _find(task_id: str):
|
|
25
|
+
for raw in await redis_client.r.lrange(DLQ_KEY, 0, -1):
|
|
26
|
+
item = json.loads(raw)
|
|
27
|
+
if item.get("task_id") == task_id:
|
|
28
|
+
return raw, item
|
|
29
|
+
raise HTTPException(status_code=404, detail="Task for particular id is not found")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async def _fresh_payload(item: dict) -> dict:
|
|
33
|
+
"""Reset a dead-lettered task so it can run again with a valid fence token."""
|
|
34
|
+
task = {k: v for k, v in item.items() if k not in _DLQ_ONLY_FIELDS}
|
|
35
|
+
task["retry_count"] = 0
|
|
36
|
+
task["prev_delay"] = 1.0
|
|
37
|
+
current_fence = await redis_client.r.get(f"fence:{task.get('task_id')}")
|
|
38
|
+
if current_fence is not None:
|
|
39
|
+
task["fence_token"] = int(current_fence)
|
|
40
|
+
return task
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _queue_ops(pipe, raw: str, task: dict):
|
|
44
|
+
task_id = task.get("task_id")
|
|
45
|
+
new_raw = json.dumps(task)
|
|
46
|
+
pipe.lrem(DLQ_KEY, 1, raw)
|
|
47
|
+
pipe.decr("stats:dlq")
|
|
48
|
+
pipe.rpush(queueing.queue_key(task.get("priority", "default")), new_raw)
|
|
49
|
+
pipe.incr("stats:pending")
|
|
50
|
+
if task_id:
|
|
51
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
52
|
+
"task_id": task_id,
|
|
53
|
+
"task_name": task.get("task_name", ""),
|
|
54
|
+
"status": queueing.QUEUED,
|
|
55
|
+
"priority": task.get("priority", "default"),
|
|
56
|
+
"retry_count": 0,
|
|
57
|
+
"payload": new_raw,
|
|
58
|
+
"error": "",
|
|
59
|
+
"note": f"Replayed from DLQ at {time.strftime('%Y-%m-%d %H:%M:%S')}",
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@router.post("/replay/{task_id}", dependencies=_operator)
|
|
64
|
+
async def replay_task(task_id: str):
|
|
65
|
+
raw, item = await _find(task_id)
|
|
66
|
+
task = await _fresh_payload(item)
|
|
67
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
68
|
+
_queue_ops(pipe, raw, task)
|
|
69
|
+
await pipe.execute()
|
|
70
|
+
await _notify_tenant("global", "dlq_updated")
|
|
71
|
+
return {"message": "Task replayed successfully", "task": task}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@router.post("/purge/{task_id}", dependencies=_operator)
|
|
75
|
+
async def purge_task(task_id: str):
|
|
76
|
+
raw, item = await _find(task_id)
|
|
77
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
78
|
+
pipe.lrem(DLQ_KEY, 1, raw)
|
|
79
|
+
pipe.decr("stats:dlq")
|
|
80
|
+
await pipe.execute()
|
|
81
|
+
await _notify_tenant("global", "dlq_updated")
|
|
82
|
+
return {"message": "task is Deleted", "task": item}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@router.post("/purge_all", dependencies=_operator)
|
|
86
|
+
async def purge_all():
|
|
87
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
88
|
+
pipe.delete(DLQ_KEY)
|
|
89
|
+
pipe.set("stats:dlq", 0)
|
|
90
|
+
await pipe.execute()
|
|
91
|
+
await _notify_tenant("global", "dlq_updated")
|
|
92
|
+
return {"message": "Entire dlq is cleared"}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@router.post("/retry_all", dependencies=_operator)
|
|
96
|
+
async def retry_all_dlq():
|
|
97
|
+
"""Move every dead-lettered task back onto its original priority queue."""
|
|
98
|
+
raws = await redis_client.r.lrange(DLQ_KEY, 0, -1)
|
|
99
|
+
if not raws:
|
|
100
|
+
return {"message": "DLQ is empty", "retried": 0}
|
|
101
|
+
|
|
102
|
+
retried = 0
|
|
103
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
104
|
+
for raw in raws:
|
|
105
|
+
try:
|
|
106
|
+
item = json.loads(raw)
|
|
107
|
+
except ValueError:
|
|
108
|
+
continue
|
|
109
|
+
_queue_ops(pipe, raw, await _fresh_payload(item))
|
|
110
|
+
retried += 1
|
|
111
|
+
await pipe.execute()
|
|
112
|
+
|
|
113
|
+
await _notify_tenant("global", "dlq_updated")
|
|
114
|
+
return {"message": f"Successfully requeued {retried} tasks", "retried": retried}
|
pytaskq/api/v1/system.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Depends
|
|
4
|
+
|
|
5
|
+
from pytaskq.core import redis_client
|
|
6
|
+
from pytaskq.core.dependencies import require
|
|
7
|
+
|
|
8
|
+
router = APIRouter(prefix="/system", tags=["system"], dependencies=[Depends(require("viewer"))])
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@router.get("/stats")
|
|
12
|
+
async def get_system_stats():
|
|
13
|
+
stats = await redis_client.r.mget(
|
|
14
|
+
"stats:pending",
|
|
15
|
+
"stats:processing",
|
|
16
|
+
"stats:completed",
|
|
17
|
+
"stats:failed",
|
|
18
|
+
"stats:delayed",
|
|
19
|
+
"stats:dlq"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
stats = [max(0, int(s)) if s else 0 for s in stats]
|
|
23
|
+
workers = await redis_client.r.zcount("active_workers", time.time(), "+inf")
|
|
24
|
+
|
|
25
|
+
return {
|
|
26
|
+
"pending": stats[0],
|
|
27
|
+
"processing": stats[1],
|
|
28
|
+
"completed": stats[2],
|
|
29
|
+
"failed": stats[3],
|
|
30
|
+
"delayed": stats[4],
|
|
31
|
+
"dlq": stats[5],
|
|
32
|
+
"active_workers": workers
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _num(value, cast=float, default=0):
|
|
37
|
+
try:
|
|
38
|
+
return cast(value)
|
|
39
|
+
except (TypeError, ValueError):
|
|
40
|
+
return default
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@router.get("/workers")
|
|
44
|
+
async def get_active_workers():
|
|
45
|
+
workers_with_scores = await redis_client.r.zrange("active_workers", 0, -1, withscores=True)
|
|
46
|
+
worker_list = []
|
|
47
|
+
now = time.time()
|
|
48
|
+
for worker_id, expire_time in workers_with_scores:
|
|
49
|
+
last_hb = expire_time - 30
|
|
50
|
+
seconds_ago = max(0, int(now - last_hb))
|
|
51
|
+
info = await redis_client.r.hgetall(f"worker:{worker_id}")
|
|
52
|
+
started_at = _num(info.get("started_at"), float, None)
|
|
53
|
+
worker_list.append({
|
|
54
|
+
"worker_id": worker_id,
|
|
55
|
+
"hostname": info.get("hostname"),
|
|
56
|
+
"pid": _num(info.get("pid"), int, None),
|
|
57
|
+
"app": info.get("app") or None,
|
|
58
|
+
"active": _num(info.get("active"), int, 0),
|
|
59
|
+
"processed": _num(info.get("processed"), int, 0),
|
|
60
|
+
"failed": _num(info.get("failed"), int, 0),
|
|
61
|
+
"cpu_workers": _num(info.get("cpu_workers"), int, None),
|
|
62
|
+
"io_workers": _num(info.get("io_workers"), int, None),
|
|
63
|
+
"uptime_seconds": int(now - started_at) if started_at else None,
|
|
64
|
+
"last_heartbeat_seconds_ago": seconds_ago,
|
|
65
|
+
"status": "healthy" if seconds_ago < 15 else ("degraded" if seconds_ago < 30 else "dead"),
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
return {"workers": worker_list, "count": len(worker_list)}
|
pytaskq/api/v1/tasks.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any, Dict, List, Literal, Optional
|
|
3
|
+
|
|
4
|
+
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
from pytaskq.core import queueing, redis_client
|
|
8
|
+
from pytaskq.core import auth
|
|
9
|
+
from pytaskq.core.dependencies import check_backpressure, require
|
|
10
|
+
|
|
11
|
+
# Reading needs "viewer"; anything that changes jobs needs "operator".
|
|
12
|
+
router = APIRouter(prefix="/task", tags=["tasks"], dependencies=[Depends(require("viewer"))])
|
|
13
|
+
list_router = APIRouter(prefix="/tasks", tags=["tasks"], dependencies=[Depends(require("viewer"))])
|
|
14
|
+
logger = logging.getLogger("api")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class EnqueueRequest(BaseModel):
|
|
18
|
+
task_name: str = Field(..., examples=["generate_csv_report"])
|
|
19
|
+
args: List[Any] = Field(default_factory=list, examples=[[100]])
|
|
20
|
+
kwargs: Dict[str, Any] = Field(default_factory=dict)
|
|
21
|
+
priority: Optional[Literal["high", "default", "low"]] = None
|
|
22
|
+
delay_seconds: float = Field(0, ge=0, description="Run after this many seconds (0 = now)")
|
|
23
|
+
on_success_url: Optional[str] = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def _validated_payload(req: EnqueueRequest) -> dict:
|
|
27
|
+
registry = {t["name"]: t for t in await queueing.registered_tasks(redis_client.r)}
|
|
28
|
+
spec = registry.get(req.task_name)
|
|
29
|
+
if registry and spec is None:
|
|
30
|
+
available = sorted(n for n, t in registry.items() if not t.get("internal"))
|
|
31
|
+
raise HTTPException(
|
|
32
|
+
status_code=404,
|
|
33
|
+
detail=f"Unknown task '{req.task_name}'. Available tasks: {', '.join(available) or 'none'}",
|
|
34
|
+
)
|
|
35
|
+
priority = req.priority or (spec or {}).get("priority") or "default"
|
|
36
|
+
return queueing.new_payload(
|
|
37
|
+
req.task_name, req.args, req.kwargs,
|
|
38
|
+
priority=priority,
|
|
39
|
+
on_success_url=req.on_success_url,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@router.post("/enqueue", dependencies=[Depends(check_backpressure)])
|
|
44
|
+
async def enqueue_task(req: EnqueueRequest, user: auth.User = Depends(require("operator"))):
|
|
45
|
+
"""Enqueue a task (immediately, or after `delay_seconds`)."""
|
|
46
|
+
payload = await _validated_payload(req)
|
|
47
|
+
result = await queueing.enqueue(redis_client.r, payload, delay_seconds=req.delay_seconds, created_by=user.username)
|
|
48
|
+
logger.info(f"Enqueued task {payload['task_id']} ({req.task_name}) priority={payload['priority']}")
|
|
49
|
+
return result
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@router.post("/schedule", dependencies=[Depends(check_backpressure)])
|
|
53
|
+
async def schedule_task(req: EnqueueRequest, delay_seconds: float = Query(60, ge=0),
|
|
54
|
+
user: auth.User = Depends(require("operator"))):
|
|
55
|
+
"""Schedule a task to run after `delay_seconds` (query parameter, default 60)."""
|
|
56
|
+
payload = await _validated_payload(req)
|
|
57
|
+
result = await queueing.enqueue(redis_client.r, payload, delay_seconds=delay_seconds, created_by=user.username)
|
|
58
|
+
logger.info(f"Scheduled task {payload['task_id']} ({req.task_name}) in {delay_seconds}s")
|
|
59
|
+
return result
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@router.get("/{task_id}")
|
|
63
|
+
async def task_result_display(task_id: str):
|
|
64
|
+
"""Fetch the current status, timings and result of a task."""
|
|
65
|
+
result = await redis_client.r.hgetall(queueing.task_key(task_id))
|
|
66
|
+
if not result:
|
|
67
|
+
raise HTTPException(status_code=404, detail="Task not found or expired.")
|
|
68
|
+
return {"result": queueing.public_view(result)}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@router.post("/{task_id}/cancel")
|
|
72
|
+
async def cancel_task(task_id: str, user: auth.User = Depends(require("operator"))):
|
|
73
|
+
"""Cancel a task that is still waiting (queued, scheduled or awaiting retry)."""
|
|
74
|
+
ok, status = await queueing.cancel(redis_client.r, task_id, by=user.username)
|
|
75
|
+
if ok:
|
|
76
|
+
return {"task_id": task_id, "status": status, "cancelled": True}
|
|
77
|
+
if status == "NotFound":
|
|
78
|
+
raise HTTPException(status_code=404, detail="Task not found or expired.")
|
|
79
|
+
raise HTTPException(status_code=409, detail=f"Task cannot be cancelled (status: {status}).")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@list_router.get("")
|
|
83
|
+
async def list_tasks(
|
|
84
|
+
status: Optional[str] = Query(None, description="Filter by status, e.g. Success, DeadLetter"),
|
|
85
|
+
task_name: Optional[str] = None,
|
|
86
|
+
limit: int = Query(50, ge=1, le=500),
|
|
87
|
+
):
|
|
88
|
+
"""Recent tasks, newest first."""
|
|
89
|
+
tasks = await queueing.recent_tasks(redis_client.r, limit=limit, status=status, task_name=task_name)
|
|
90
|
+
return {"tasks": tasks, "count": len(tasks)}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@list_router.get("/registered")
|
|
94
|
+
async def list_registered_tasks(include_internal: bool = False):
|
|
95
|
+
"""Tasks advertised by running workers (with their parameters, used by the dashboard)."""
|
|
96
|
+
tasks = await queueing.registered_tasks(redis_client.r)
|
|
97
|
+
if not include_internal:
|
|
98
|
+
tasks = [t for t in tasks if not t.get("internal")]
|
|
99
|
+
return {"tasks": tasks}
|