pytaskq 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq/__init__.py +6 -0
- pytaskq/__main__.py +4 -0
- pytaskq/api/router.py +15 -0
- pytaskq/api/v1/auth.py +235 -0
- pytaskq/api/v1/dashboard.py +99 -0
- pytaskq/api/v1/dlq.py +114 -0
- pytaskq/api/v1/system.py +68 -0
- pytaskq/api/v1/tasks.py +99 -0
- pytaskq/app.py +324 -0
- pytaskq/cli.py +250 -0
- pytaskq/core/auth.py +535 -0
- pytaskq/core/config.py +52 -0
- pytaskq/core/dependencies.py +106 -0
- pytaskq/core/exceptions.py +12 -0
- pytaskq/core/idempotency.py +45 -0
- pytaskq/core/queueing.py +282 -0
- pytaskq/core/redis_client.py +19 -0
- pytaskq/main.py +124 -0
- pytaskq/net.py +67 -0
- pytaskq/schemas/requests.py +32 -0
- pytaskq/services/dashboard_service.py +42 -0
- pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq/static/favicon.svg +1 -0
- pytaskq/static/index.html +15 -0
- pytaskq/task_registery.py +144 -0
- pytaskq/tracing.py +98 -0
- pytaskq/worker.py +636 -0
- pytaskq-0.3.0.dist-info/METADATA +262 -0
- pytaskq-0.3.0.dist-info/RECORD +34 -0
- pytaskq-0.3.0.dist-info/WHEEL +5 -0
- pytaskq-0.3.0.dist-info/entry_points.txt +2 -0
- pytaskq-0.3.0.dist-info/licenses/LICENSE +21 -0
- pytaskq-0.3.0.dist-info/top_level.txt +1 -0
pytaskq/core/auth.py
ADDED
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pytaskq.core.auth — users, roles, sessions, invites and API tokens
|
|
3
|
+
===================================================================
|
|
4
|
+
|
|
5
|
+
Everything lives in Redis next to the jobs, so a self-hosted install needs no
|
|
6
|
+
extra database and survives restarts as long as Redis persists (Docker Compose
|
|
7
|
+
already mounts a volume for it).
|
|
8
|
+
|
|
9
|
+
Security model
|
|
10
|
+
--------------
|
|
11
|
+
* Passwords are hashed with scrypt (stdlib, memory-hard, per-user random salt).
|
|
12
|
+
* Session cookies, invite links and API tokens are random 256-bit secrets.
|
|
13
|
+
Redis stores only their SHA-256, so a Redis dump does not hand out logins.
|
|
14
|
+
* Roles are ordered: viewer < operator < admin.
|
|
15
|
+
* The first admin is created with a one-time setup code printed in the server
|
|
16
|
+
log (or headlessly from PYTASKQ_ADMIN_USER / PYTASKQ_ADMIN_PASSWORD).
|
|
17
|
+
* Repeated failed logins are throttled per username and per IP.
|
|
18
|
+
|
|
19
|
+
Redis keys
|
|
20
|
+
----------
|
|
21
|
+
auth:users SET usernames
|
|
22
|
+
auth:user:{username} HASH username, role, password_hash, created_at, …
|
|
23
|
+
auth:session:{sha} STR username (TTL = session lifetime)
|
|
24
|
+
auth:user_sessions:{username} SET session hashes (to revoke all on reset)
|
|
25
|
+
auth:invites SET invite hashes
|
|
26
|
+
auth:invite:{sha} HASH id, role, created_by, … (TTL = 7 days)
|
|
27
|
+
auth:token:{sha} HASH id, username, name, created_at, last_used_at
|
|
28
|
+
auth:user_tokens:{username} SET token hashes
|
|
29
|
+
auth:setup_code STR one-time code for creating the first admin
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import asyncio
|
|
35
|
+
import base64
|
|
36
|
+
import hashlib
|
|
37
|
+
import hmac
|
|
38
|
+
import re
|
|
39
|
+
import secrets
|
|
40
|
+
import time
|
|
41
|
+
from dataclasses import asdict, dataclass
|
|
42
|
+
from typing import Optional
|
|
43
|
+
|
|
44
|
+
from pytaskq.core.config import settings
|
|
45
|
+
|
|
46
|
+
ROLES = ("viewer", "operator", "admin")
|
|
47
|
+
ROLE_LEVEL = {role: i for i, role in enumerate(ROLES, start=1)}
|
|
48
|
+
|
|
49
|
+
USERNAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{1,31}$")
|
|
50
|
+
MIN_PASSWORD = 8
|
|
51
|
+
MAX_PASSWORD = 256
|
|
52
|
+
|
|
53
|
+
INVITE_TTL = 7 * 24 * 3600
|
|
54
|
+
LOGIN_WINDOW = 15 * 60
|
|
55
|
+
MAX_FAILS_PER_USER = 10
|
|
56
|
+
MAX_FAILS_PER_IP = 30
|
|
57
|
+
|
|
58
|
+
TOKEN_PREFIX = "ptq_"
|
|
59
|
+
|
|
60
|
+
K_USERS = "auth:users"
|
|
61
|
+
K_INVITES = "auth:invites"
|
|
62
|
+
K_SETUP_CODE = "auth:setup_code"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class AuthError(Exception):
|
|
66
|
+
"""A user-facing auth problem. `status` maps to the HTTP status code."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, message: str, status: int = 400):
|
|
69
|
+
super().__init__(message)
|
|
70
|
+
self.message = message
|
|
71
|
+
self.status = status
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class User:
|
|
76
|
+
username: str
|
|
77
|
+
role: str
|
|
78
|
+
created_at: float = 0.0
|
|
79
|
+
created_by: str = ""
|
|
80
|
+
last_login_at: float = 0.0
|
|
81
|
+
disabled: bool = False
|
|
82
|
+
via: str = "session" # session | token | master_key | auth_off
|
|
83
|
+
|
|
84
|
+
def can(self, role: str) -> bool:
|
|
85
|
+
return not self.disabled and ROLE_LEVEL.get(self.role, 0) >= ROLE_LEVEL[role]
|
|
86
|
+
|
|
87
|
+
def public(self) -> dict:
|
|
88
|
+
d = asdict(self)
|
|
89
|
+
d.pop("via", None)
|
|
90
|
+
return d
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _sha(secret: str) -> str:
|
|
94
|
+
return hashlib.sha256(secret.encode()).hexdigest()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _k_user(username: str) -> str:
|
|
98
|
+
return f"auth:user:{username}"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ── password hashing ─────────────────────────────────────────────────────────
|
|
102
|
+
|
|
103
|
+
_SCRYPT = dict(n=2 ** 14, r=8, p=1, dklen=64)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _hash_password_sync(password: str) -> str:
|
|
107
|
+
salt = secrets.token_bytes(16)
|
|
108
|
+
digest = hashlib.scrypt(password.encode(), salt=salt, maxmem=64 * 1024 * 1024, **_SCRYPT)
|
|
109
|
+
b64 = lambda b: base64.b64encode(b).decode() # noqa: E731
|
|
110
|
+
return f"scrypt${_SCRYPT['n']}${_SCRYPT['r']}${_SCRYPT['p']}${b64(salt)}${b64(digest)}"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _verify_password_sync(password: str, encoded: str) -> bool:
|
|
114
|
+
try:
|
|
115
|
+
algo, n, r, p, salt_b64, digest_b64 = encoded.split("$")
|
|
116
|
+
if algo != "scrypt":
|
|
117
|
+
return False
|
|
118
|
+
expected = base64.b64decode(digest_b64)
|
|
119
|
+
digest = hashlib.scrypt(
|
|
120
|
+
password.encode(), salt=base64.b64decode(salt_b64),
|
|
121
|
+
n=int(n), r=int(r), p=int(p), dklen=len(expected), maxmem=64 * 1024 * 1024,
|
|
122
|
+
)
|
|
123
|
+
return hmac.compare_digest(digest, expected)
|
|
124
|
+
except (ValueError, TypeError):
|
|
125
|
+
return False
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
async def hash_password(password: str) -> str:
|
|
129
|
+
# scrypt is deliberately slow (~50 ms); keep it off the event loop
|
|
130
|
+
return await asyncio.to_thread(_hash_password_sync, password)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
async def verify_password(password: str, encoded: str) -> bool:
|
|
134
|
+
return await asyncio.to_thread(_verify_password_sync, password, encoded)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# Used to spend the same time on unknown usernames as on wrong passwords,
|
|
138
|
+
# so response timing doesn't reveal which usernames exist.
|
|
139
|
+
_DUMMY_HASH = _hash_password_sync(secrets.token_hex(8))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ── validation ───────────────────────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
def normalize_username(username: str) -> str:
|
|
145
|
+
name = (username or "").strip().lower()
|
|
146
|
+
if not USERNAME_RE.match(name):
|
|
147
|
+
raise AuthError(
|
|
148
|
+
"Usernames are 2–32 characters: lowercase letters, numbers, dots, dashes or underscores, "
|
|
149
|
+
"starting with a letter or number."
|
|
150
|
+
)
|
|
151
|
+
return name
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def check_password_rules(password: str) -> None:
|
|
155
|
+
if not isinstance(password, str) or len(password) < MIN_PASSWORD:
|
|
156
|
+
raise AuthError(f"Passwords need at least {MIN_PASSWORD} characters.")
|
|
157
|
+
if len(password) > MAX_PASSWORD:
|
|
158
|
+
raise AuthError(f"Passwords can be at most {MAX_PASSWORD} characters.")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def check_role(role: str) -> str:
|
|
162
|
+
if role not in ROLES:
|
|
163
|
+
raise AuthError(f"Role must be one of: {', '.join(ROLES)}.")
|
|
164
|
+
return role
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# ── users ────────────────────────────────────────────────────────────────────
|
|
168
|
+
|
|
169
|
+
def _user_from_hash(h: dict) -> User:
|
|
170
|
+
return User(
|
|
171
|
+
username=h["username"],
|
|
172
|
+
role=h.get("role", "viewer"),
|
|
173
|
+
created_at=float(h.get("created_at") or 0),
|
|
174
|
+
created_by=h.get("created_by", ""),
|
|
175
|
+
last_login_at=float(h.get("last_login_at") or 0),
|
|
176
|
+
disabled=h.get("disabled") == "1",
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
async def user_count(r) -> int:
|
|
181
|
+
return await r.scard(K_USERS)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
async def get_user(r, username: str) -> Optional[User]:
|
|
185
|
+
h = await r.hgetall(_k_user(username))
|
|
186
|
+
return _user_from_hash(h) if h else None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
async def list_users(r) -> list[User]:
|
|
190
|
+
names = sorted(await r.smembers(K_USERS))
|
|
191
|
+
users = []
|
|
192
|
+
for name in names:
|
|
193
|
+
u = await get_user(r, name)
|
|
194
|
+
if u:
|
|
195
|
+
users.append(u)
|
|
196
|
+
return users
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
async def create_user(r, username: str, password: str, role: str, created_by: str = "") -> User:
|
|
200
|
+
username = normalize_username(username)
|
|
201
|
+
check_password_rules(password)
|
|
202
|
+
check_role(role)
|
|
203
|
+
pw_hash = await hash_password(password)
|
|
204
|
+
now = time.time()
|
|
205
|
+
# HSETNX on the username field makes "create" atomic: two people accepting
|
|
206
|
+
# invites with the same name can't overwrite each other.
|
|
207
|
+
if not await r.hsetnx(_k_user(username), "username", username):
|
|
208
|
+
raise AuthError(f"The username '{username}' is taken.", 409)
|
|
209
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
210
|
+
pipe.hset(_k_user(username), mapping={
|
|
211
|
+
"role": role, "password_hash": pw_hash, "created_at": now,
|
|
212
|
+
"created_by": created_by, "disabled": "0",
|
|
213
|
+
})
|
|
214
|
+
pipe.sadd(K_USERS, username)
|
|
215
|
+
await pipe.execute()
|
|
216
|
+
return User(username=username, role=role, created_at=now, created_by=created_by)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
async def _active_admins(r) -> list[str]:
|
|
220
|
+
return [u.username for u in await list_users(r) if u.role == "admin" and not u.disabled]
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
async def update_user(r, username: str, *, role: Optional[str] = None,
|
|
224
|
+
disabled: Optional[bool] = None, acting: Optional[User] = None) -> User:
|
|
225
|
+
user = await get_user(r, username)
|
|
226
|
+
if not user:
|
|
227
|
+
raise AuthError("User not found.", 404)
|
|
228
|
+
losing_admin = user.role == "admin" and not user.disabled and (
|
|
229
|
+
(role is not None and role != "admin") or disabled is True
|
|
230
|
+
)
|
|
231
|
+
if losing_admin and await _active_admins(r) == [username]:
|
|
232
|
+
raise AuthError("This is the only active admin. Make someone else an admin first.", 409)
|
|
233
|
+
changes = {}
|
|
234
|
+
if role is not None:
|
|
235
|
+
changes["role"] = check_role(role)
|
|
236
|
+
if disabled is not None:
|
|
237
|
+
changes["disabled"] = "1" if disabled else "0"
|
|
238
|
+
if changes:
|
|
239
|
+
await r.hset(_k_user(username), mapping=changes)
|
|
240
|
+
if disabled:
|
|
241
|
+
await revoke_all_sessions(r, username)
|
|
242
|
+
return await get_user(r, username)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
async def delete_user(r, username: str) -> None:
|
|
246
|
+
user = await get_user(r, username)
|
|
247
|
+
if not user:
|
|
248
|
+
raise AuthError("User not found.", 404)
|
|
249
|
+
if user.role == "admin" and not user.disabled and await _active_admins(r) == [username]:
|
|
250
|
+
raise AuthError("This is the only active admin. Make someone else an admin first.", 409)
|
|
251
|
+
await revoke_all_sessions(r, username)
|
|
252
|
+
for token_sha in await r.smembers(f"auth:user_tokens:{username}"):
|
|
253
|
+
await r.delete(f"auth:token:{token_sha}")
|
|
254
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
255
|
+
pipe.delete(_k_user(username), f"auth:user_tokens:{username}")
|
|
256
|
+
pipe.srem(K_USERS, username)
|
|
257
|
+
await pipe.execute()
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
async def set_password(r, username: str, new_password: str, keep_session_sha: Optional[str] = None) -> None:
|
|
261
|
+
check_password_rules(new_password)
|
|
262
|
+
if not await r.exists(_k_user(username)):
|
|
263
|
+
raise AuthError("User not found.", 404)
|
|
264
|
+
await r.hset(_k_user(username), mapping={
|
|
265
|
+
"password_hash": await hash_password(new_password),
|
|
266
|
+
"password_changed_at": time.time(),
|
|
267
|
+
})
|
|
268
|
+
await revoke_all_sessions(r, username, keep=keep_session_sha)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
# ── login throttling ─────────────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
async def _failures(r, username: str, ip: str) -> tuple[int, int]:
|
|
274
|
+
u, i = await r.mget(f"auth:fail:user:{username}", f"auth:fail:ip:{ip}")
|
|
275
|
+
return int(u or 0), int(i or 0)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
async def _record_failure(r, username: str, ip: str) -> None:
|
|
279
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
280
|
+
for key in (f"auth:fail:user:{username}", f"auth:fail:ip:{ip}"):
|
|
281
|
+
pipe.incr(key)
|
|
282
|
+
pipe.expire(key, LOGIN_WINDOW)
|
|
283
|
+
await pipe.execute()
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
async def authenticate(r, username: str, password: str, ip: str = "unknown") -> User:
|
|
287
|
+
"""Check a username/password. Raises AuthError (generic message) on failure."""
|
|
288
|
+
name = (username or "").strip().lower()
|
|
289
|
+
by_user, by_ip = await _failures(r, name, ip)
|
|
290
|
+
if by_user >= MAX_FAILS_PER_USER or by_ip >= MAX_FAILS_PER_IP:
|
|
291
|
+
raise AuthError("Too many failed sign-in attempts. Try again in 15 minutes.", 429)
|
|
292
|
+
|
|
293
|
+
h = await r.hgetall(_k_user(name)) if USERNAME_RE.match(name) else {}
|
|
294
|
+
ok = await verify_password(password or "", h.get("password_hash") or _DUMMY_HASH)
|
|
295
|
+
if not h or not ok or h.get("disabled") == "1":
|
|
296
|
+
await _record_failure(r, name, ip)
|
|
297
|
+
raise AuthError("Wrong username or password.", 401)
|
|
298
|
+
|
|
299
|
+
await r.delete(f"auth:fail:user:{name}")
|
|
300
|
+
return _user_from_hash(h)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# ── sessions ─────────────────────────────────────────────────────────────────
|
|
304
|
+
|
|
305
|
+
def session_ttl() -> int:
|
|
306
|
+
return max(1, settings.SESSION_TTL_DAYS) * 86400
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
async def create_session(r, username: str) -> str:
|
|
310
|
+
token = secrets.token_urlsafe(32)
|
|
311
|
+
sha = _sha(token)
|
|
312
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
313
|
+
pipe.set(f"auth:session:{sha}", username, ex=session_ttl())
|
|
314
|
+
pipe.sadd(f"auth:user_sessions:{username}", sha)
|
|
315
|
+
pipe.hset(_k_user(username), "last_login_at", time.time()) # every sign-in path goes through here
|
|
316
|
+
await pipe.execute()
|
|
317
|
+
return token
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
async def user_for_session(r, token: str) -> Optional[User]:
|
|
321
|
+
if not token:
|
|
322
|
+
return None
|
|
323
|
+
username = await r.get(f"auth:session:{_sha(token)}")
|
|
324
|
+
if not username:
|
|
325
|
+
return None
|
|
326
|
+
user = await get_user(r, username)
|
|
327
|
+
if not user or user.disabled:
|
|
328
|
+
return None
|
|
329
|
+
return user
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
async def end_session(r, token: str) -> None:
|
|
333
|
+
if not token:
|
|
334
|
+
return
|
|
335
|
+
sha = _sha(token)
|
|
336
|
+
username = await r.get(f"auth:session:{sha}")
|
|
337
|
+
await r.delete(f"auth:session:{sha}")
|
|
338
|
+
if username:
|
|
339
|
+
await r.srem(f"auth:user_sessions:{username}", sha)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
async def revoke_all_sessions(r, username: str, keep: Optional[str] = None) -> None:
|
|
343
|
+
key = f"auth:user_sessions:{username}"
|
|
344
|
+
for sha in await r.smembers(key):
|
|
345
|
+
if sha != keep:
|
|
346
|
+
await r.delete(f"auth:session:{sha}")
|
|
347
|
+
await r.srem(key, sha)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def session_sha(token: str) -> str:
|
|
351
|
+
return _sha(token) if token else ""
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# ── API tokens ───────────────────────────────────────────────────────────────
|
|
355
|
+
|
|
356
|
+
async def create_api_token(r, username: str, name: str) -> tuple[str, dict]:
|
|
357
|
+
name = (name or "").strip()[:60] or "API token"
|
|
358
|
+
token = TOKEN_PREFIX + secrets.token_urlsafe(32)
|
|
359
|
+
sha = _sha(token)
|
|
360
|
+
info = {"id": sha[:12], "username": username, "name": name, "created_at": time.time(), "last_used_at": 0}
|
|
361
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
362
|
+
pipe.hset(f"auth:token:{sha}", mapping=info)
|
|
363
|
+
pipe.sadd(f"auth:user_tokens:{username}", sha)
|
|
364
|
+
await pipe.execute()
|
|
365
|
+
return token, _token_view(info)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _token_view(h: dict) -> dict:
|
|
369
|
+
return {
|
|
370
|
+
"id": h.get("id"),
|
|
371
|
+
"name": h.get("name"),
|
|
372
|
+
"created_at": float(h.get("created_at") or 0),
|
|
373
|
+
"last_used_at": float(h.get("last_used_at") or 0) or None,
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
async def list_api_tokens(r, username: str) -> list[dict]:
|
|
378
|
+
out = []
|
|
379
|
+
for sha in await r.smembers(f"auth:user_tokens:{username}"):
|
|
380
|
+
h = await r.hgetall(f"auth:token:{sha}")
|
|
381
|
+
if h:
|
|
382
|
+
out.append(_token_view(h))
|
|
383
|
+
return sorted(out, key=lambda t: t["created_at"], reverse=True)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
async def revoke_api_token(r, username: str, token_id: str) -> bool:
|
|
387
|
+
for sha in await r.smembers(f"auth:user_tokens:{username}"):
|
|
388
|
+
if sha.startswith(token_id) and len(token_id) >= 8:
|
|
389
|
+
await r.delete(f"auth:token:{sha}")
|
|
390
|
+
await r.srem(f"auth:user_tokens:{username}", sha)
|
|
391
|
+
return True
|
|
392
|
+
return False
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
async def user_for_api_token(r, token: str) -> Optional[User]:
|
|
396
|
+
if not token or not token.startswith(TOKEN_PREFIX):
|
|
397
|
+
return None
|
|
398
|
+
sha = _sha(token)
|
|
399
|
+
h = await r.hgetall(f"auth:token:{sha}")
|
|
400
|
+
if not h:
|
|
401
|
+
return None
|
|
402
|
+
user = await get_user(r, h["username"])
|
|
403
|
+
if not user or user.disabled:
|
|
404
|
+
return None
|
|
405
|
+
await r.hset(f"auth:token:{sha}", "last_used_at", time.time())
|
|
406
|
+
user.via = "token"
|
|
407
|
+
return user
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
# ── invites ──────────────────────────────────────────────────────────────────
|
|
411
|
+
|
|
412
|
+
async def create_invite(r, role: str, created_by: str) -> tuple[str, dict]:
|
|
413
|
+
check_role(role)
|
|
414
|
+
token = secrets.token_urlsafe(24)
|
|
415
|
+
sha = _sha(token)
|
|
416
|
+
now = time.time()
|
|
417
|
+
info = {"id": sha[:12], "role": role, "created_by": created_by,
|
|
418
|
+
"created_at": now, "expires_at": now + INVITE_TTL}
|
|
419
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
420
|
+
pipe.hset(f"auth:invite:{sha}", mapping=info)
|
|
421
|
+
pipe.expire(f"auth:invite:{sha}", INVITE_TTL)
|
|
422
|
+
pipe.sadd(K_INVITES, sha)
|
|
423
|
+
await pipe.execute()
|
|
424
|
+
return token, _invite_view(info)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _invite_view(h: dict) -> dict:
|
|
428
|
+
return {
|
|
429
|
+
"id": h.get("id"),
|
|
430
|
+
"role": h.get("role"),
|
|
431
|
+
"created_by": h.get("created_by"),
|
|
432
|
+
"created_at": float(h.get("created_at") or 0),
|
|
433
|
+
"expires_at": float(h.get("expires_at") or 0),
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
async def get_invite(r, token: str) -> Optional[dict]:
|
|
438
|
+
h = await r.hgetall(f"auth:invite:{_sha(token or '')}")
|
|
439
|
+
return _invite_view(h) if h else None
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
async def list_invites(r) -> list[dict]:
|
|
443
|
+
out = []
|
|
444
|
+
for sha in await r.smembers(K_INVITES):
|
|
445
|
+
h = await r.hgetall(f"auth:invite:{sha}")
|
|
446
|
+
if h:
|
|
447
|
+
out.append(_invite_view(h))
|
|
448
|
+
else:
|
|
449
|
+
await r.srem(K_INVITES, sha) # expired
|
|
450
|
+
return sorted(out, key=lambda i: i["created_at"], reverse=True)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
async def revoke_invite(r, invite_id: str) -> bool:
|
|
454
|
+
for sha in await r.smembers(K_INVITES):
|
|
455
|
+
if sha.startswith(invite_id) and len(invite_id) >= 8:
|
|
456
|
+
await r.delete(f"auth:invite:{sha}")
|
|
457
|
+
await r.srem(K_INVITES, sha)
|
|
458
|
+
return True
|
|
459
|
+
return False
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
async def accept_invite(r, token: str, username: str, password: str) -> User:
|
|
463
|
+
sha = _sha(token or "")
|
|
464
|
+
h = await r.hgetall(f"auth:invite:{sha}")
|
|
465
|
+
if not h:
|
|
466
|
+
raise AuthError("This invite link is invalid or has expired. Ask an admin for a new one.", 404)
|
|
467
|
+
# Validate everything before consuming the invite, so a typo doesn't burn it.
|
|
468
|
+
username = normalize_username(username)
|
|
469
|
+
check_password_rules(password)
|
|
470
|
+
if await r.exists(_k_user(username)):
|
|
471
|
+
raise AuthError(f"The username '{username}' is taken.", 409)
|
|
472
|
+
# Consume atomically: only one person can use a link.
|
|
473
|
+
if not await r.delete(f"auth:invite:{sha}"):
|
|
474
|
+
raise AuthError("This invite link has already been used.", 409)
|
|
475
|
+
await r.srem(K_INVITES, sha)
|
|
476
|
+
return await create_user(r, username, password, h["role"], created_by=h.get("created_by", ""))
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
# ── first-run setup ──────────────────────────────────────────────────────────
|
|
480
|
+
|
|
481
|
+
async def setup_required(r) -> bool:
|
|
482
|
+
return settings.auth_enabled and await user_count(r) == 0
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
async def ensure_setup_code(r) -> Optional[str]:
|
|
486
|
+
"""Return the current one-time setup code, creating it if needed (None if setup is done)."""
|
|
487
|
+
if await user_count(r) > 0:
|
|
488
|
+
return None
|
|
489
|
+
code = "-".join(secrets.token_hex(3).upper() for _ in range(3)) # e.g. 3F9A1C-0B7E22-9D41F0
|
|
490
|
+
await r.set(K_SETUP_CODE, code, nx=True)
|
|
491
|
+
return await r.get(K_SETUP_CODE)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
async def complete_setup(r, code: str, username: str, password: str) -> User:
|
|
495
|
+
if await user_count(r) > 0:
|
|
496
|
+
raise AuthError("Setup is already complete. Sign in instead.", 409)
|
|
497
|
+
expected = await r.get(K_SETUP_CODE)
|
|
498
|
+
given = (code or "").strip().upper()
|
|
499
|
+
if not expected or not hmac.compare_digest(given, expected):
|
|
500
|
+
raise AuthError("That setup code is not right. Copy it from the PyTaskQ server log.", 403)
|
|
501
|
+
# Lock so two browsers can't both finish setup at the same moment
|
|
502
|
+
if not await r.set("auth:setup_lock", "1", nx=True, ex=30):
|
|
503
|
+
raise AuthError("Setup is already in progress.", 409)
|
|
504
|
+
try:
|
|
505
|
+
user = await create_user(r, username, password, "admin", created_by="setup")
|
|
506
|
+
await r.delete(K_SETUP_CODE)
|
|
507
|
+
return user
|
|
508
|
+
finally:
|
|
509
|
+
await r.delete("auth:setup_lock")
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
async def bootstrap_from_env(r) -> Optional[User]:
|
|
513
|
+
"""Create the first admin from PYTASKQ_ADMIN_USER / PYTASKQ_ADMIN_PASSWORD if no users exist."""
|
|
514
|
+
if not (settings.PYTASKQ_ADMIN_USER and settings.PYTASKQ_ADMIN_PASSWORD):
|
|
515
|
+
return None
|
|
516
|
+
if await user_count(r) > 0:
|
|
517
|
+
return None
|
|
518
|
+
user = await create_user(r, settings.PYTASKQ_ADMIN_USER, settings.PYTASKQ_ADMIN_PASSWORD,
|
|
519
|
+
"admin", created_by="env")
|
|
520
|
+
await r.delete(K_SETUP_CODE)
|
|
521
|
+
return user
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
# Pseudo-users for non-interactive access
|
|
525
|
+
def local_admin() -> User:
|
|
526
|
+
return User(username="local", role="admin", via="auth_off")
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def master_key_user() -> User:
|
|
530
|
+
return User(username="master-key", role="admin", via="master_key")
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def matches_master_key(token: str) -> bool:
|
|
534
|
+
key = settings.MASTER_KEY
|
|
535
|
+
return bool(key) and len(key) >= 16 and hmac.compare_digest(token.encode(), key.encode())
|
pytaskq/core/config.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Settings(BaseSettings):
|
|
5
|
+
"""
|
|
6
|
+
Every PyTaskQ setting can be overridden with an environment variable of the
|
|
7
|
+
same name (or a line in `.env`). See `.env.example` for documentation.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
|
11
|
+
|
|
12
|
+
# ── Redis ────────────────────────────────────────────────────────────────
|
|
13
|
+
# REDIS_URL wins when set (e.g. redis://:password@host:6379/0 or rediss://…)
|
|
14
|
+
REDIS_URL: str = ""
|
|
15
|
+
REDIS_HOST: str = "localhost"
|
|
16
|
+
REDIS_PORT: int = 6379
|
|
17
|
+
REDIS_DB: int = 0
|
|
18
|
+
REDIS_PASSWORD: str = ""
|
|
19
|
+
|
|
20
|
+
# ── Queue behaviour ──────────────────────────────────────────────────────
|
|
21
|
+
QUEUE_CAPACITY: int = 500
|
|
22
|
+
TASK_TIMEOUT: float = 300 # default per-task timeout (seconds, 0 = none)
|
|
23
|
+
MAX_RETRIES: int = 3 # default retries before a task is dead-lettered
|
|
24
|
+
RESULT_TTL: int = 86400 # how long task status/results are kept (seconds)
|
|
25
|
+
HISTORY_LIMIT: int = 500 # how many recent tasks the dashboard can list
|
|
26
|
+
|
|
27
|
+
# ── Authentication (self-hosting) ────────────────────────────────────────
|
|
28
|
+
PYTASKQ_AUTH: str = "on" # "off" disables login (local development only)
|
|
29
|
+
SESSION_TTL_DAYS: int = 30 # how long a dashboard sign-in lasts
|
|
30
|
+
COOKIE_SECURE: str = "auto" # "auto" = Secure cookie when served over HTTPS
|
|
31
|
+
PYTASKQ_ADMIN_USER: str = "" # optional: create the first admin on startup…
|
|
32
|
+
PYTASKQ_ADMIN_PASSWORD: str = "" # …instead of using the setup code
|
|
33
|
+
MASTER_KEY: str = "" # optional: Bearer key with admin rights (automation)
|
|
34
|
+
|
|
35
|
+
# ── API ──────────────────────────────────────────────────────────────────
|
|
36
|
+
ALLOWED_ORIGINS: str = "*"
|
|
37
|
+
RATE_LIMIT_PER_MINUTE: int = 100
|
|
38
|
+
IDEMPOTENCY_TTL: int = 86400
|
|
39
|
+
IDEMPOTENCY_PENDING_TTL: int = 30
|
|
40
|
+
DEDUP_TTL: int = 5
|
|
41
|
+
ENVIRONMENT: str = "development"
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def auth_enabled(self) -> bool:
|
|
45
|
+
return self.PYTASKQ_AUTH.strip().lower() not in ("off", "false", "0", "no", "disabled")
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def allowed_origins_list(self) -> list[str]:
|
|
49
|
+
return [o.strip() for o in self.ALLOWED_ORIGINS.split(",") if o.strip()]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
settings = Settings()
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from fastapi import HTTPException, Request
|
|
3
|
+
from pytaskq.core import redis_client
|
|
4
|
+
from pytaskq.core.config import settings
|
|
5
|
+
|
|
6
|
+
def get_client_ip(req: Request) -> str:
|
|
7
|
+
"""Extract true client IP, respecting proxy headers if behind Docker/Ngrok/Nginx"""
|
|
8
|
+
forwarded = req.headers.get("x-forwarded-for")
|
|
9
|
+
if forwarded:
|
|
10
|
+
return forwarded.split(",")[0].strip()
|
|
11
|
+
return req.client.host
|
|
12
|
+
|
|
13
|
+
async def check_backpressure():
|
|
14
|
+
queue_len = (
|
|
15
|
+
await redis_client.r.llen("queue:high")
|
|
16
|
+
+ await redis_client.r.llen("queue:default")
|
|
17
|
+
+ await redis_client.r.llen("queue:low")
|
|
18
|
+
)
|
|
19
|
+
if queue_len >= settings.QUEUE_CAPACITY:
|
|
20
|
+
raise HTTPException(
|
|
21
|
+
status_code=429,
|
|
22
|
+
detail="Server is busy. Please retry after a few seconds.",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
async def rate_limiter():
|
|
26
|
+
"""
|
|
27
|
+
CHECK-ONLY rate limiter — does NOT increment the counter.
|
|
28
|
+
"""
|
|
29
|
+
current_minute = int(time.time() / 60)
|
|
30
|
+
key = f"rate_limit:global:{current_minute}"
|
|
31
|
+
count = int(await redis_client.r.get(key) or 0)
|
|
32
|
+
if count >= settings.RATE_LIMIT_PER_MINUTE:
|
|
33
|
+
raise HTTPException(status_code=429, detail="Too many Requests. Please wait a minute")
|
|
34
|
+
|
|
35
|
+
async def _incr_rate_limit() -> None:
|
|
36
|
+
"""Atomically increment the rate limit counter for a confirmed fresh request."""
|
|
37
|
+
current_minute = int(time.time() / 60)
|
|
38
|
+
key = f"rate_limit:global:{current_minute}"
|
|
39
|
+
count = await redis_client.r.incr(key)
|
|
40
|
+
if count == 1:
|
|
41
|
+
await redis_client.r.expire(key, 60)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# ── Authentication & roles ────────────────────────────────────────────────────
|
|
45
|
+
from urllib.parse import urlparse
|
|
46
|
+
|
|
47
|
+
from pytaskq.core import auth
|
|
48
|
+
|
|
49
|
+
SESSION_COOKIE = "pytaskq_session"
|
|
50
|
+
_SAFE_METHODS = {"GET", "HEAD", "OPTIONS"}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _same_origin(origin: str, host_header: str) -> bool:
|
|
54
|
+
"""Cross-site request forgery guard for cookie-authenticated writes."""
|
|
55
|
+
if not origin:
|
|
56
|
+
return True # non-browser clients (curl, SDKs) don't send Origin
|
|
57
|
+
if origin in [o for o in settings.allowed_origins_list if o != "*"]:
|
|
58
|
+
return True
|
|
59
|
+
origin_host = urlparse(origin).hostname or ""
|
|
60
|
+
request_host = (host_header or "").rsplit(":", 1)[0].strip("[]")
|
|
61
|
+
return bool(origin_host) and origin_host == request_host
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
async def authenticate_request(headers, cookies, method: str = "GET") -> auth.User:
|
|
65
|
+
"""Resolve who is calling: API token / master key, session cookie, or auth disabled."""
|
|
66
|
+
if not settings.auth_enabled:
|
|
67
|
+
return auth.local_admin()
|
|
68
|
+
|
|
69
|
+
authorization = headers.get("authorization") or ""
|
|
70
|
+
if authorization.lower().startswith("bearer "):
|
|
71
|
+
token = authorization[7:].strip()
|
|
72
|
+
if auth.matches_master_key(token):
|
|
73
|
+
return auth.master_key_user()
|
|
74
|
+
user = await auth.user_for_api_token(redis_client.r, token)
|
|
75
|
+
if user:
|
|
76
|
+
return user
|
|
77
|
+
raise HTTPException(status_code=401, detail="Invalid or revoked API token.")
|
|
78
|
+
|
|
79
|
+
user = await auth.user_for_session(redis_client.r, cookies.get(SESSION_COOKIE, ""))
|
|
80
|
+
if not user:
|
|
81
|
+
raise HTTPException(status_code=401, detail="Sign in to continue.")
|
|
82
|
+
if method.upper() not in _SAFE_METHODS:
|
|
83
|
+
origin = headers.get("origin") or headers.get("referer") or ""
|
|
84
|
+
if not _same_origin(origin, headers.get("host", "")):
|
|
85
|
+
raise HTTPException(status_code=403, detail="Cross-site request blocked.")
|
|
86
|
+
return user
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
async def current_user(request: Request) -> auth.User:
|
|
90
|
+
user = await authenticate_request(request.headers, request.cookies, request.method)
|
|
91
|
+
request.state.user = user
|
|
92
|
+
return user
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def require(role: str):
|
|
96
|
+
"""Dependency factory: `Depends(require("operator"))` → 403 unless the user has that role."""
|
|
97
|
+
if role not in auth.ROLE_LEVEL:
|
|
98
|
+
raise ValueError(f"unknown role {role}")
|
|
99
|
+
|
|
100
|
+
async def checker(request: Request) -> auth.User:
|
|
101
|
+
user = await current_user(request)
|
|
102
|
+
if not user.can(role):
|
|
103
|
+
raise HTTPException(status_code=403, detail=f"This needs the {role} role (you are {user.role}).")
|
|
104
|
+
return user
|
|
105
|
+
|
|
106
|
+
return checker
|