pytaskq 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq/__init__.py +6 -0
- pytaskq/__main__.py +4 -0
- pytaskq/api/router.py +15 -0
- pytaskq/api/v1/auth.py +235 -0
- pytaskq/api/v1/dashboard.py +99 -0
- pytaskq/api/v1/dlq.py +114 -0
- pytaskq/api/v1/system.py +68 -0
- pytaskq/api/v1/tasks.py +99 -0
- pytaskq/app.py +324 -0
- pytaskq/cli.py +250 -0
- pytaskq/core/auth.py +535 -0
- pytaskq/core/config.py +52 -0
- pytaskq/core/dependencies.py +106 -0
- pytaskq/core/exceptions.py +12 -0
- pytaskq/core/idempotency.py +45 -0
- pytaskq/core/queueing.py +282 -0
- pytaskq/core/redis_client.py +19 -0
- pytaskq/main.py +124 -0
- pytaskq/net.py +67 -0
- pytaskq/schemas/requests.py +32 -0
- pytaskq/services/dashboard_service.py +42 -0
- pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq/static/favicon.svg +1 -0
- pytaskq/static/index.html +15 -0
- pytaskq/task_registery.py +144 -0
- pytaskq/tracing.py +98 -0
- pytaskq/worker.py +636 -0
- pytaskq-0.3.0.dist-info/METADATA +262 -0
- pytaskq-0.3.0.dist-info/RECORD +34 -0
- pytaskq-0.3.0.dist-info/WHEEL +5 -0
- pytaskq-0.3.0.dist-info/entry_points.txt +2 -0
- pytaskq-0.3.0.dist-info/licenses/LICENSE +21 -0
- pytaskq-0.3.0.dist-info/top_level.txt +1 -0
pytaskq/app.py
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pytaskq.app — the PyTaskQ SDK
|
|
3
|
+
==============================
|
|
4
|
+
|
|
5
|
+
from pytaskq import PyTaskQ
|
|
6
|
+
|
|
7
|
+
queue = PyTaskQ()
|
|
8
|
+
|
|
9
|
+
@queue.task # plain decorator works
|
|
10
|
+
def send_welcome_email(user_id): ...
|
|
11
|
+
|
|
12
|
+
@queue.task(type="cpu", max_retries=5, timeout=60, priority="high")
|
|
13
|
+
def render_report(report_id, fmt="pdf"): ...
|
|
14
|
+
|
|
15
|
+
@queue.task # async def is detected automatically
|
|
16
|
+
async def call_api(url): ...
|
|
17
|
+
|
|
18
|
+
@queue.task(cron="*/5 * * * *") # runs every 5 minutes on the fleet
|
|
19
|
+
def cleanup(): ...
|
|
20
|
+
|
|
21
|
+
# Enqueue from async code
|
|
22
|
+
await render_report.delay(42, fmt="csv")
|
|
23
|
+
await render_report.apply_async(args=[42], priority="low", countdown=30)
|
|
24
|
+
|
|
25
|
+
# …or from plain sync code (scripts, Django/Flask views)
|
|
26
|
+
render_report.delay_sync(42)
|
|
27
|
+
|
|
28
|
+
# Inspect results
|
|
29
|
+
await queue.status(task_id)
|
|
30
|
+
await queue.wait(task_id, timeout=30)
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import asyncio
|
|
36
|
+
import hashlib
|
|
37
|
+
import hmac
|
|
38
|
+
import inspect
|
|
39
|
+
import json
|
|
40
|
+
import os
|
|
41
|
+
import time
|
|
42
|
+
from typing import Any, Callable, Optional
|
|
43
|
+
|
|
44
|
+
from pytaskq.core import queueing
|
|
45
|
+
from pytaskq.core.config import settings
|
|
46
|
+
from pytaskq.net import SAFE_OPENER, assert_safe_url
|
|
47
|
+
|
|
48
|
+
TASK_TYPES = ("cpu", "io", "async")
|
|
49
|
+
|
|
50
|
+
# Option names `delay()` treats as queue options rather than task kwargs —
|
|
51
|
+
# unless the task function itself declares a parameter with that name.
|
|
52
|
+
_DELAY_OPTIONS = ("priority", "on_success_url")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _redis():
|
|
56
|
+
# Looked up at call time so tests (and apps) can swap the client.
|
|
57
|
+
from pytaskq.core import redis_client
|
|
58
|
+
return redis_client.r
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _describe_params(func: Callable) -> list[dict]:
|
|
62
|
+
"""JSON-able description of the task's parameters (drives dashboard forms)."""
|
|
63
|
+
params = []
|
|
64
|
+
try:
|
|
65
|
+
sig = inspect.signature(func)
|
|
66
|
+
except (TypeError, ValueError):
|
|
67
|
+
return params
|
|
68
|
+
for p in sig.parameters.values():
|
|
69
|
+
if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD):
|
|
70
|
+
params.append({"name": p.name, "kind": "var_positional" if p.kind == p.VAR_POSITIONAL else "var_keyword",
|
|
71
|
+
"required": False})
|
|
72
|
+
continue
|
|
73
|
+
info: dict[str, Any] = {"name": p.name, "required": p.default is p.empty}
|
|
74
|
+
if p.default is not p.empty:
|
|
75
|
+
try:
|
|
76
|
+
json.dumps(p.default)
|
|
77
|
+
info["default"] = p.default
|
|
78
|
+
except (TypeError, ValueError):
|
|
79
|
+
info["default"] = repr(p.default)
|
|
80
|
+
if p.annotation is not p.empty:
|
|
81
|
+
ann = p.annotation
|
|
82
|
+
info["annotation"] = ann if isinstance(ann, str) else getattr(ann, "__name__", str(ann))
|
|
83
|
+
params.append(info)
|
|
84
|
+
return params
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def describe_tasks(tasks: dict[str, dict]) -> dict[str, dict]:
|
|
88
|
+
"""JSON-able description of a TASKS registry dict."""
|
|
89
|
+
out = {}
|
|
90
|
+
for task_name, entry in tasks.items():
|
|
91
|
+
fn = entry["handler"]
|
|
92
|
+
doc = inspect.getdoc(fn) or ""
|
|
93
|
+
out[task_name] = {
|
|
94
|
+
"type": entry.get("type", "cpu"),
|
|
95
|
+
"cron": entry.get("cron"),
|
|
96
|
+
"priority": entry.get("priority", "default"),
|
|
97
|
+
"max_retries": entry.get("max_retries") if entry.get("max_retries") is not None else settings.MAX_RETRIES,
|
|
98
|
+
"timeout": entry.get("timeout") if entry.get("timeout") is not None else settings.TASK_TIMEOUT,
|
|
99
|
+
"doc": doc.split("\n\n")[0].strip(),
|
|
100
|
+
"params": _describe_params(fn),
|
|
101
|
+
"internal": task_name.startswith("_"),
|
|
102
|
+
}
|
|
103
|
+
return out
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class PyTaskQ:
|
|
107
|
+
"""A registry of background tasks plus the helpers to enqueue them."""
|
|
108
|
+
|
|
109
|
+
def __init__(
|
|
110
|
+
self,
|
|
111
|
+
name: str = "pytaskq",
|
|
112
|
+
*,
|
|
113
|
+
default_priority: str = "default",
|
|
114
|
+
max_retries: Optional[int] = None,
|
|
115
|
+
timeout: Optional[float] = None,
|
|
116
|
+
include_builtins: bool = True,
|
|
117
|
+
):
|
|
118
|
+
if default_priority not in queueing.PRIORITIES:
|
|
119
|
+
raise ValueError(f"default_priority must be one of {queueing.PRIORITIES}")
|
|
120
|
+
self.name = name
|
|
121
|
+
self.default_priority = default_priority
|
|
122
|
+
self.max_retries = max_retries
|
|
123
|
+
self.timeout = timeout
|
|
124
|
+
self.TASKS: dict[str, dict] = {}
|
|
125
|
+
if include_builtins:
|
|
126
|
+
self.task(name="_deliver_webhook", type="io")(_deliver_webhook)
|
|
127
|
+
|
|
128
|
+
# ── registration ─────────────────────────────────────────────────────────
|
|
129
|
+
def task(
|
|
130
|
+
self,
|
|
131
|
+
func: Optional[Callable] = None,
|
|
132
|
+
*,
|
|
133
|
+
name: Optional[str] = None,
|
|
134
|
+
type: Optional[str] = None,
|
|
135
|
+
cron: Optional[str] = None,
|
|
136
|
+
priority: Optional[str] = None,
|
|
137
|
+
max_retries: Optional[int] = None,
|
|
138
|
+
timeout: Optional[float] = None,
|
|
139
|
+
):
|
|
140
|
+
"""
|
|
141
|
+
Register a function as a background task. Usable as `@queue.task` or
|
|
142
|
+
`@queue.task(...)`.
|
|
143
|
+
|
|
144
|
+
type "cpu" (process pool), "io" (thread pool) or "async" (event loop).
|
|
145
|
+
Defaults to "async" for `async def`, otherwise "cpu".
|
|
146
|
+
cron Standard 5-field cron expression; the fleet enqueues it on schedule.
|
|
147
|
+
priority Default queue for this task: "high", "default" or "low".
|
|
148
|
+
max_retries Attempts after the first failure before dead-lettering (default 3).
|
|
149
|
+
timeout Seconds before the attempt is abandoned (default TASK_TIMEOUT).
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
if isinstance(func, str): # legacy positional form: @queue.task("io")
|
|
153
|
+
type, func = func, None
|
|
154
|
+
|
|
155
|
+
def decorator(fn: Callable) -> Callable:
|
|
156
|
+
is_async = inspect.iscoroutinefunction(fn)
|
|
157
|
+
task_type = type or ("async" if is_async else "cpu")
|
|
158
|
+
if task_type not in TASK_TYPES:
|
|
159
|
+
raise ValueError(f"type must be one of {TASK_TYPES}, got {task_type!r}")
|
|
160
|
+
if is_async and task_type != "async":
|
|
161
|
+
raise ValueError(f"{fn.__name__} is `async def`; use type='async' (or leave type unset)")
|
|
162
|
+
if task_type == "async" and not is_async:
|
|
163
|
+
raise ValueError(f"{fn.__name__} has type='async' but is not an `async def` function")
|
|
164
|
+
prio = priority or self.default_priority
|
|
165
|
+
if prio not in queueing.PRIORITIES:
|
|
166
|
+
raise ValueError(f"priority must be one of {queueing.PRIORITIES}, got {prio!r}")
|
|
167
|
+
if cron is not None:
|
|
168
|
+
import croniter
|
|
169
|
+
if not croniter.croniter.is_valid(cron):
|
|
170
|
+
raise ValueError(f"Invalid cron expression for {fn.__name__}: {cron!r}")
|
|
171
|
+
if max_retries is not None and max_retries < 0:
|
|
172
|
+
raise ValueError("max_retries must be >= 0")
|
|
173
|
+
|
|
174
|
+
task_name = name or fn.__name__
|
|
175
|
+
if task_name in self.TASKS and self.TASKS[task_name]["handler"] is not fn:
|
|
176
|
+
raise ValueError(f"A task named {task_name!r} is already registered")
|
|
177
|
+
|
|
178
|
+
retries = max_retries if max_retries is not None else self.max_retries
|
|
179
|
+
limit = timeout if timeout is not None else self.timeout
|
|
180
|
+
self.TASKS[task_name] = {
|
|
181
|
+
"handler": fn,
|
|
182
|
+
"type": task_type,
|
|
183
|
+
"cron": cron,
|
|
184
|
+
"priority": prio,
|
|
185
|
+
"max_retries": retries,
|
|
186
|
+
"timeout": limit,
|
|
187
|
+
}
|
|
188
|
+
own_params = set(inspect.signature(fn).parameters)
|
|
189
|
+
|
|
190
|
+
def _split(kwargs: dict) -> tuple[dict, dict]:
|
|
191
|
+
opts = {k: kwargs.pop(k) for k in _DELAY_OPTIONS if k in kwargs and k not in own_params}
|
|
192
|
+
return kwargs, opts
|
|
193
|
+
|
|
194
|
+
async def apply_async(args=None, kwargs=None, *, priority: Optional[str] = None,
|
|
195
|
+
countdown: float = 0, on_success_url: Optional[str] = None,
|
|
196
|
+
task_id: Optional[str] = None) -> dict:
|
|
197
|
+
payload = queueing.new_payload(
|
|
198
|
+
task_name, args, kwargs,
|
|
199
|
+
priority=priority or prio,
|
|
200
|
+
on_success_url=on_success_url,
|
|
201
|
+
max_retries=retries,
|
|
202
|
+
timeout=limit,
|
|
203
|
+
task_id=task_id,
|
|
204
|
+
)
|
|
205
|
+
return await queueing.enqueue(_redis(), payload, delay_seconds=countdown)
|
|
206
|
+
|
|
207
|
+
async def delay(*args, **kwargs) -> dict:
|
|
208
|
+
kwargs, opts = _split(kwargs)
|
|
209
|
+
return await apply_async(args, kwargs, **opts)
|
|
210
|
+
|
|
211
|
+
async def schedule(*args, delay_seconds: float = 60, **kwargs) -> dict:
|
|
212
|
+
kwargs, opts = _split(kwargs)
|
|
213
|
+
return await apply_async(args, kwargs, countdown=delay_seconds, **opts)
|
|
214
|
+
|
|
215
|
+
def delay_sync(*args, **kwargs) -> dict:
|
|
216
|
+
"""Enqueue from synchronous code (no running event loop)."""
|
|
217
|
+
kwargs, opts = _split(kwargs)
|
|
218
|
+
return _run_sync(lambda r: queueing.enqueue(
|
|
219
|
+
r,
|
|
220
|
+
queueing.new_payload(task_name, args, kwargs, priority=opts.get("priority") or prio,
|
|
221
|
+
on_success_url=opts.get("on_success_url"),
|
|
222
|
+
max_retries=retries, timeout=limit),
|
|
223
|
+
))
|
|
224
|
+
|
|
225
|
+
fn.delay = delay
|
|
226
|
+
fn.schedule = schedule
|
|
227
|
+
fn.apply_async = apply_async
|
|
228
|
+
fn.delay_sync = delay_sync
|
|
229
|
+
fn.task_name = task_name
|
|
230
|
+
return fn
|
|
231
|
+
|
|
232
|
+
if func is not None and callable(func):
|
|
233
|
+
return decorator(func)
|
|
234
|
+
return decorator
|
|
235
|
+
|
|
236
|
+
# ── introspection ────────────────────────────────────────────────────────
|
|
237
|
+
def specs(self) -> dict[str, dict]:
|
|
238
|
+
"""Describe registered tasks (published to Redis for the dashboard)."""
|
|
239
|
+
return describe_tasks(self.TASKS)
|
|
240
|
+
|
|
241
|
+
async def publish(self) -> None:
|
|
242
|
+
"""Advertise this app's tasks so the dashboard and API can see them."""
|
|
243
|
+
await queueing.publish_registry(_redis(), self.specs())
|
|
244
|
+
|
|
245
|
+
# ── results ──────────────────────────────────────────────────────────────
|
|
246
|
+
async def status(self, task_id: str) -> Optional[dict]:
|
|
247
|
+
"""Current status hash of a task (None if unknown or expired)."""
|
|
248
|
+
meta = await _redis().hgetall(queueing.task_key(task_id))
|
|
249
|
+
return queueing.public_view(meta) if meta else None
|
|
250
|
+
|
|
251
|
+
async def wait(self, task_id: str, timeout: float = 30, poll_interval: float = 0.25) -> dict:
|
|
252
|
+
"""Wait until the task reaches a final state; raises TimeoutError otherwise."""
|
|
253
|
+
deadline = time.monotonic() + timeout
|
|
254
|
+
while True:
|
|
255
|
+
info = await self.status(task_id)
|
|
256
|
+
if info and info.get("status") in queueing.FINAL_STATES:
|
|
257
|
+
return info
|
|
258
|
+
if time.monotonic() >= deadline:
|
|
259
|
+
raise TimeoutError(f"Task {task_id} did not finish within {timeout}s")
|
|
260
|
+
await asyncio.sleep(poll_interval)
|
|
261
|
+
|
|
262
|
+
async def cancel(self, task_id: str) -> bool:
|
|
263
|
+
"""Cancel a task that has not started yet."""
|
|
264
|
+
ok, _ = await queueing.cancel(_redis(), task_id)
|
|
265
|
+
return ok
|
|
266
|
+
|
|
267
|
+
def __repr__(self) -> str:
|
|
268
|
+
return f"<PyTaskQ {self.name!r} tasks={sorted(self.TASKS)}>"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _run_sync(coro_factory):
|
|
272
|
+
try:
|
|
273
|
+
asyncio.get_running_loop()
|
|
274
|
+
except RuntimeError:
|
|
275
|
+
pass
|
|
276
|
+
else:
|
|
277
|
+
raise RuntimeError("delay_sync() was called inside a running event loop; use `await task.delay(...)` instead")
|
|
278
|
+
|
|
279
|
+
async def runner():
|
|
280
|
+
from pytaskq.core.redis_client import make_client
|
|
281
|
+
client = make_client()
|
|
282
|
+
try:
|
|
283
|
+
return await coro_factory(client)
|
|
284
|
+
finally:
|
|
285
|
+
await client.aclose()
|
|
286
|
+
|
|
287
|
+
return asyncio.run(runner())
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# ── Built-in task: reliable webhook delivery ────────────────────────────────────
|
|
291
|
+
# Enqueued by the worker after a task with `on_success_url` succeeds. It goes
|
|
292
|
+
# through the queue like any other task, so it gets retries and the DLQ.
|
|
293
|
+
def _deliver_webhook(url: str, task_id: str, task_name: str, result: str):
|
|
294
|
+
"""Deliver a signed POST to a task's on_success_url."""
|
|
295
|
+
assert_safe_url(url)
|
|
296
|
+
|
|
297
|
+
payload = json.dumps({
|
|
298
|
+
"event": "task.success",
|
|
299
|
+
"task_id": task_id,
|
|
300
|
+
"task_name": task_name,
|
|
301
|
+
"result": result,
|
|
302
|
+
}).encode("utf-8")
|
|
303
|
+
|
|
304
|
+
# Optional HMAC-SHA256 signing if WEBHOOK_SECRET is set
|
|
305
|
+
secret = os.environ.get("WEBHOOK_SECRET", "")
|
|
306
|
+
signature = hmac.new(secret.encode(), payload, hashlib.sha256).hexdigest() if secret else "unsigned"
|
|
307
|
+
|
|
308
|
+
import urllib.request
|
|
309
|
+
req = urllib.request.Request(
|
|
310
|
+
url,
|
|
311
|
+
data=payload,
|
|
312
|
+
headers={
|
|
313
|
+
"Content-Type": "application/json",
|
|
314
|
+
"User-Agent": "PyTaskQ-Webhook/1.0",
|
|
315
|
+
"X-PyTaskQ-Signature": f"sha256={signature}",
|
|
316
|
+
"X-PyTaskQ-Task-ID": task_id,
|
|
317
|
+
},
|
|
318
|
+
method="POST",
|
|
319
|
+
)
|
|
320
|
+
with SAFE_OPENER.open(req, timeout=10) as resp:
|
|
321
|
+
status = resp.status
|
|
322
|
+
if status < 200 or status >= 300:
|
|
323
|
+
raise RuntimeError(f"Webhook delivery failed: target returned HTTP {status}")
|
|
324
|
+
return {"delivered_to": url, "status_code": status}
|
pytaskq/cli.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PyTaskQ command line
|
|
3
|
+
====================
|
|
4
|
+
|
|
5
|
+
pytaskq worker --app myapp.tasks:queue [--cpu-workers N] [--io-workers N] [--timeout S]
|
|
6
|
+
pytaskq dashboard [--app myapp.tasks:queue] [--host 127.0.0.1] [--port 8000]
|
|
7
|
+
pytaskq tasks --app myapp.tasks:queue
|
|
8
|
+
pytaskq ping
|
|
9
|
+
pytaskq users list | create NAME | reset-password NAME | set-role NAME ROLE | delete NAME | setup-code
|
|
10
|
+
|
|
11
|
+
`--app` can also come from the PYTASKQ_APP environment variable.
|
|
12
|
+
`users` talks to Redis directly, so it works even when you're locked out of the dashboard.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import asyncio
|
|
17
|
+
import importlib
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
from dotenv import load_dotenv
|
|
23
|
+
|
|
24
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
25
|
+
log = logging.getLogger("pytaskq")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CLIError(Exception):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def load_app(spec: str):
|
|
33
|
+
"""Import `module:attribute` and return the PyTaskQ instance."""
|
|
34
|
+
if not spec:
|
|
35
|
+
raise CLIError("No app given. Use --app module:queue (or set PYTASKQ_APP), e.g. --app example_app:queue")
|
|
36
|
+
sys.path.insert(0, os.getcwd())
|
|
37
|
+
module_name, _, app_name = spec.partition(":")
|
|
38
|
+
app_name = app_name or "queue"
|
|
39
|
+
try:
|
|
40
|
+
module = importlib.import_module(module_name)
|
|
41
|
+
except ImportError as e:
|
|
42
|
+
raise CLIError(f"Could not import module '{module_name}': {e}") from e
|
|
43
|
+
try:
|
|
44
|
+
app = getattr(module, app_name)
|
|
45
|
+
except AttributeError:
|
|
46
|
+
raise CLIError(f"Could not find '{app_name}' in module '{module_name}'") from None
|
|
47
|
+
if not hasattr(app, "TASKS"):
|
|
48
|
+
raise CLIError(f"'{spec}' is not a PyTaskQ instance (no TASKS registry)")
|
|
49
|
+
return app
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def cmd_worker(args):
|
|
53
|
+
app = load_app(args.app)
|
|
54
|
+
from pytaskq import worker
|
|
55
|
+
if args.cpu_workers:
|
|
56
|
+
worker.CPU_WORKERS = args.cpu_workers
|
|
57
|
+
if args.io_workers:
|
|
58
|
+
worker.IO_WORKERS = args.io_workers
|
|
59
|
+
if args.timeout is not None:
|
|
60
|
+
worker.TASK_TIMEOUT = args.timeout
|
|
61
|
+
worker.set_tasks(app.TASKS, app_name=args.app)
|
|
62
|
+
log.info(f"Starting PyTaskQ worker for app: {args.app}")
|
|
63
|
+
asyncio.run(worker.consumer_task())
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def cmd_dashboard(args):
|
|
67
|
+
import uvicorn
|
|
68
|
+
if args.app:
|
|
69
|
+
os.environ["PYTASKQ_APP"] = args.app
|
|
70
|
+
load_app(args.app) # fail fast with a clear message
|
|
71
|
+
url = f"http://{'localhost' if args.host in ('0.0.0.0', '127.0.0.1') else args.host}:{args.port}"
|
|
72
|
+
log.info(f"PyTaskQ dashboard → {url} (API docs: {url}/docs)")
|
|
73
|
+
uvicorn.run("pytaskq.main:app", host=args.host, port=args.port, reload=args.reload, log_level="info")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def cmd_tasks(args):
|
|
77
|
+
app = load_app(args.app)
|
|
78
|
+
specs = app.specs()
|
|
79
|
+
rows = [(n, s) for n, s in sorted(specs.items()) if not s["internal"]]
|
|
80
|
+
if not rows:
|
|
81
|
+
print("No tasks registered.")
|
|
82
|
+
return
|
|
83
|
+
print(f"{len(rows)} task(s) in {args.app}:\n")
|
|
84
|
+
for name, s in rows:
|
|
85
|
+
params = ", ".join(
|
|
86
|
+
p["name"] + ("" if p.get("required") else f"={p.get('default')!r}") for p in s["params"]
|
|
87
|
+
)
|
|
88
|
+
extras = [s["type"], f"priority={s['priority']}", f"retries={s['max_retries']}", f"timeout={s['timeout']:g}s"]
|
|
89
|
+
if s["cron"]:
|
|
90
|
+
extras.append(f"cron='{s['cron']}'")
|
|
91
|
+
print(f" {name}({params})")
|
|
92
|
+
print(f" {' · '.join(extras)}")
|
|
93
|
+
if s["doc"]:
|
|
94
|
+
print(f" {s['doc']}")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def cmd_ping(args):
|
|
98
|
+
from pytaskq.core.redis_client import make_client
|
|
99
|
+
from pytaskq.core.config import settings
|
|
100
|
+
|
|
101
|
+
async def go():
|
|
102
|
+
client = make_client()
|
|
103
|
+
try:
|
|
104
|
+
await client.ping()
|
|
105
|
+
workers = await client.zcard("active_workers")
|
|
106
|
+
return workers
|
|
107
|
+
finally:
|
|
108
|
+
await client.aclose()
|
|
109
|
+
|
|
110
|
+
target = settings.REDIS_URL or f"{settings.REDIS_HOST}:{settings.REDIS_PORT}"
|
|
111
|
+
try:
|
|
112
|
+
workers = asyncio.run(go())
|
|
113
|
+
except Exception as e:
|
|
114
|
+
raise CLIError(f"Cannot reach Redis at {target}: {e}") from e
|
|
115
|
+
print(f"Redis OK at {target} — {workers} worker(s) registered")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _with_redis(fn):
|
|
119
|
+
"""Run an async function with a fresh Redis client (CLI helpers)."""
|
|
120
|
+
from pytaskq.core.redis_client import make_client
|
|
121
|
+
|
|
122
|
+
async def go():
|
|
123
|
+
client = make_client()
|
|
124
|
+
try:
|
|
125
|
+
return await fn(client)
|
|
126
|
+
finally:
|
|
127
|
+
await client.aclose()
|
|
128
|
+
|
|
129
|
+
return asyncio.run(go())
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _ask_password(given: str = "") -> str:
|
|
133
|
+
if given:
|
|
134
|
+
return given
|
|
135
|
+
import getpass
|
|
136
|
+
first = getpass.getpass("New password: ")
|
|
137
|
+
if first != getpass.getpass("Repeat password: "):
|
|
138
|
+
raise CLIError("Passwords don't match.")
|
|
139
|
+
return first
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def cmd_users(args):
|
|
143
|
+
from datetime import datetime
|
|
144
|
+
|
|
145
|
+
from pytaskq.core import auth
|
|
146
|
+
|
|
147
|
+
action = args.action
|
|
148
|
+
try:
|
|
149
|
+
if action == "list":
|
|
150
|
+
users = _with_redis(auth.list_users)
|
|
151
|
+
if not users:
|
|
152
|
+
print("No users yet. Open the dashboard and use the setup code, or run: pytaskq users create NAME --role admin")
|
|
153
|
+
return
|
|
154
|
+
print(f"{'USERNAME':<24}{'ROLE':<10}{'STATUS':<10}LAST SIGN-IN")
|
|
155
|
+
for u in users:
|
|
156
|
+
last = datetime.fromtimestamp(u.last_login_at).strftime("%Y-%m-%d %H:%M") if u.last_login_at else "never"
|
|
157
|
+
print(f"{u.username:<24}{u.role:<10}{'disabled' if u.disabled else 'active':<10}{last}")
|
|
158
|
+
elif action == "create":
|
|
159
|
+
password = _ask_password(args.password)
|
|
160
|
+
user = _with_redis(lambda r: auth.create_user(r, args.username, password, args.role, created_by="cli"))
|
|
161
|
+
print(f"Created {user.role} '{user.username}'.")
|
|
162
|
+
elif action == "reset-password":
|
|
163
|
+
name = auth.normalize_username(args.username)
|
|
164
|
+
password = _ask_password(args.password)
|
|
165
|
+
_with_redis(lambda r: auth.set_password(r, name, password))
|
|
166
|
+
print(f"Password for '{name}' changed; their other sessions were signed out.")
|
|
167
|
+
elif action == "set-role":
|
|
168
|
+
name = auth.normalize_username(args.username)
|
|
169
|
+
user = _with_redis(lambda r: auth.update_user(r, name, role=args.role))
|
|
170
|
+
print(f"'{user.username}' is now {user.role}.")
|
|
171
|
+
elif action == "delete":
|
|
172
|
+
name = auth.normalize_username(args.username)
|
|
173
|
+
_with_redis(lambda r: auth.delete_user(r, name))
|
|
174
|
+
print(f"Deleted '{name}'.")
|
|
175
|
+
elif action == "setup-code":
|
|
176
|
+
code = _with_redis(auth.ensure_setup_code)
|
|
177
|
+
print(code if code else "Setup is already complete (an account exists). Use 'pytaskq users list'.")
|
|
178
|
+
except auth.AuthError as e:
|
|
179
|
+
raise CLIError(e.message) from e
|
|
180
|
+
except CLIError:
|
|
181
|
+
raise
|
|
182
|
+
except Exception as e: # connection problems etc.
|
|
183
|
+
raise CLIError(f"Could not reach Redis: {e}") from e
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
187
|
+
parser = argparse.ArgumentParser(prog="pytaskq", description="PyTaskQ — background jobs for Python")
|
|
188
|
+
sub = parser.add_subparsers(dest="command")
|
|
189
|
+
default_app = os.environ.get("PYTASKQ_APP", "")
|
|
190
|
+
|
|
191
|
+
w = sub.add_parser("worker", help="Start a worker that executes tasks")
|
|
192
|
+
w.add_argument("--app", default=default_app, help="PyTaskQ instance, e.g. example_app:queue")
|
|
193
|
+
w.add_argument("--cpu-workers", type=int, default=0, help="Processes for type='cpu' tasks (default: cores-1)")
|
|
194
|
+
w.add_argument("--io-workers", type=int, default=0, help="Threads for type='io' tasks (default: cores*4)")
|
|
195
|
+
w.add_argument("--timeout", type=float, default=None, help="Default task timeout in seconds (0 = none)")
|
|
196
|
+
w.set_defaults(func=cmd_worker)
|
|
197
|
+
|
|
198
|
+
d = sub.add_parser("dashboard", help="Start the web dashboard + REST API")
|
|
199
|
+
d.add_argument("--app", default=default_app, help="Optional: show this app's tasks before any worker starts")
|
|
200
|
+
d.add_argument("--host", default="127.0.0.1")
|
|
201
|
+
d.add_argument("--port", type=int, default=8000)
|
|
202
|
+
d.add_argument("--reload", action="store_true", help="Auto-reload on code changes (development)")
|
|
203
|
+
d.set_defaults(func=cmd_dashboard)
|
|
204
|
+
|
|
205
|
+
t = sub.add_parser("tasks", help="List the tasks registered in an app")
|
|
206
|
+
t.add_argument("--app", default=default_app)
|
|
207
|
+
t.set_defaults(func=cmd_tasks)
|
|
208
|
+
|
|
209
|
+
p = sub.add_parser("ping", help="Check the Redis connection")
|
|
210
|
+
p.set_defaults(func=cmd_ping)
|
|
211
|
+
|
|
212
|
+
u = sub.add_parser("users", help="Manage dashboard accounts (works even when locked out)")
|
|
213
|
+
us = u.add_subparsers(dest="action", required=True)
|
|
214
|
+
us.add_parser("list", help="List accounts")
|
|
215
|
+
c = us.add_parser("create", help="Create an account")
|
|
216
|
+
c.add_argument("username")
|
|
217
|
+
c.add_argument("--role", choices=["viewer", "operator", "admin"], default="viewer")
|
|
218
|
+
c.add_argument("--password", default="", help="Omit to be prompted (keeps it out of shell history)")
|
|
219
|
+
rp = us.add_parser("reset-password", help="Set a new password for an account")
|
|
220
|
+
rp.add_argument("username")
|
|
221
|
+
rp.add_argument("--password", default="")
|
|
222
|
+
sr = us.add_parser("set-role", help="Change an account's role")
|
|
223
|
+
sr.add_argument("username")
|
|
224
|
+
sr.add_argument("role", choices=["viewer", "operator", "admin"])
|
|
225
|
+
d = us.add_parser("delete", help="Delete an account")
|
|
226
|
+
d.add_argument("username")
|
|
227
|
+
us.add_parser("setup-code", help="Show the first-run setup code")
|
|
228
|
+
u.set_defaults(func=cmd_users)
|
|
229
|
+
return parser
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def main(argv=None):
|
|
233
|
+
load_dotenv()
|
|
234
|
+
parser = build_parser()
|
|
235
|
+
args = parser.parse_args(argv)
|
|
236
|
+
if not getattr(args, "func", None):
|
|
237
|
+
parser.print_help()
|
|
238
|
+
return 1
|
|
239
|
+
try:
|
|
240
|
+
args.func(args)
|
|
241
|
+
except CLIError as e:
|
|
242
|
+
log.error(str(e))
|
|
243
|
+
sys.exit(1)
|
|
244
|
+
except KeyboardInterrupt:
|
|
245
|
+
pass
|
|
246
|
+
return 0
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
if __name__ == "__main__":
|
|
250
|
+
main()
|