pytaskq 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq/__init__.py +6 -0
- pytaskq/__main__.py +4 -0
- pytaskq/api/router.py +15 -0
- pytaskq/api/v1/auth.py +235 -0
- pytaskq/api/v1/dashboard.py +99 -0
- pytaskq/api/v1/dlq.py +114 -0
- pytaskq/api/v1/system.py +68 -0
- pytaskq/api/v1/tasks.py +99 -0
- pytaskq/app.py +324 -0
- pytaskq/cli.py +250 -0
- pytaskq/core/auth.py +535 -0
- pytaskq/core/config.py +52 -0
- pytaskq/core/dependencies.py +106 -0
- pytaskq/core/exceptions.py +12 -0
- pytaskq/core/idempotency.py +45 -0
- pytaskq/core/queueing.py +282 -0
- pytaskq/core/redis_client.py +19 -0
- pytaskq/main.py +124 -0
- pytaskq/net.py +67 -0
- pytaskq/schemas/requests.py +32 -0
- pytaskq/services/dashboard_service.py +42 -0
- pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq/static/favicon.svg +1 -0
- pytaskq/static/index.html +15 -0
- pytaskq/task_registery.py +144 -0
- pytaskq/tracing.py +98 -0
- pytaskq/worker.py +636 -0
- pytaskq-0.3.0.dist-info/METADATA +262 -0
- pytaskq-0.3.0.dist-info/RECORD +34 -0
- pytaskq-0.3.0.dist-info/WHEEL +5 -0
- pytaskq-0.3.0.dist-info/entry_points.txt +2 -0
- pytaskq-0.3.0.dist-info/licenses/LICENSE +21 -0
- pytaskq-0.3.0.dist-info/top_level.txt +1 -0
pytaskq/worker.py
ADDED
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pytaskq.worker — the task-executing process
|
|
3
|
+
============================================
|
|
4
|
+
|
|
5
|
+
Start one (or many, on many machines) with:
|
|
6
|
+
|
|
7
|
+
pytaskq worker --app myproject.tasks:queue
|
|
8
|
+
|
|
9
|
+
Each worker pops tasks in strict priority order (high → default → low), runs
|
|
10
|
+
them in a process pool (type="cpu"), a thread pool (type="io") or directly on
|
|
11
|
+
the event loop (async def), and records status, timings and results in Redis.
|
|
12
|
+
Background loops (only on the elected leader) handle cron schedules, delayed
|
|
13
|
+
retries and recovery of tasks from crashed workers.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
import functools
|
|
18
|
+
import inspect
|
|
19
|
+
import json
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import random
|
|
23
|
+
import signal
|
|
24
|
+
import socket
|
|
25
|
+
import time
|
|
26
|
+
import uuid
|
|
27
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
|
28
|
+
from datetime import datetime
|
|
29
|
+
|
|
30
|
+
import croniter
|
|
31
|
+
from opentelemetry import trace
|
|
32
|
+
|
|
33
|
+
from pytaskq.core import queueing
|
|
34
|
+
from pytaskq.core import redis_client
|
|
35
|
+
from pytaskq.core.config import settings
|
|
36
|
+
from pytaskq.schemas.requests import Taskloader, Taskresult # noqa: F401 (Taskresult kept for API compat)
|
|
37
|
+
from pytaskq.tracing import extract_trace_context, init_tracer, inject_trace_context # noqa: F401
|
|
38
|
+
|
|
39
|
+
TASKS = {}
|
|
40
|
+
APP_NAME = ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def set_tasks(tasks_dict, app_name: str = ""):
|
|
44
|
+
"""Register the tasks this worker may execute (called by the CLI)."""
|
|
45
|
+
global APP_NAME
|
|
46
|
+
TASKS.update(tasks_dict)
|
|
47
|
+
if "_deliver_webhook" not in TASKS:
|
|
48
|
+
# Webhooks must work for every app, even one built from a plain dict.
|
|
49
|
+
from pytaskq.app import _deliver_webhook
|
|
50
|
+
TASKS["_deliver_webhook"] = {"handler": _deliver_webhook, "type": "io"}
|
|
51
|
+
if app_name:
|
|
52
|
+
APP_NAME = app_name
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
logging.basicConfig(
|
|
56
|
+
level=logging.INFO,
|
|
57
|
+
format="%(asctime)s [%(levelname)s] [%(processName)s] %(message)s",
|
|
58
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
59
|
+
)
|
|
60
|
+
logger = logging.getLogger("worker")
|
|
61
|
+
|
|
62
|
+
HOSTNAME = socket.gethostname()
|
|
63
|
+
WORKER_ID = f"{HOSTNAME}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
|
64
|
+
TASK_TIMEOUT = float(os.getenv("TASK_TIMEOUT", str(settings.TASK_TIMEOUT)))
|
|
65
|
+
HEARTBEAT_TTL = 30
|
|
66
|
+
|
|
67
|
+
_TOTAL_CORES = os.cpu_count() or 1
|
|
68
|
+
_RESERVED_CORES = int(os.getenv("WORKER_RESERVED_CORES", "1"))
|
|
69
|
+
CPU_WORKERS = int(os.getenv("WORKER_CPU_WORKERS", "0")) or max(1, _TOTAL_CORES - _RESERVED_CORES)
|
|
70
|
+
IO_WORKERS = int(os.getenv("WORKER_IO_WORKERS", "0")) or max(4, _TOTAL_CORES * 4)
|
|
71
|
+
CONCURRENCY = CPU_WORKERS + IO_WORKERS
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class TaskTimeoutError(Exception):
|
|
75
|
+
"""Raised when a task exceeds its timeout."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class UnknownTaskError(Exception):
|
|
79
|
+
"""Raised when a payload names a task this worker does not know."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
shutdown_event = asyncio.Event()
|
|
83
|
+
active_tasks = set()
|
|
84
|
+
tracer = init_tracer("pytaskq-worker")
|
|
85
|
+
|
|
86
|
+
FENCE_SCRIPT = """
|
|
87
|
+
local current = redis.call('get', KEYS[1])
|
|
88
|
+
if current == false or current == ARGV[1] then
|
|
89
|
+
redis.call('hset', KEYS[2], 'task_id', ARGV[2], 'status', ARGV[3], 'result', ARGV[4])
|
|
90
|
+
redis.call('expire', KEYS[2], 86400)
|
|
91
|
+
return 1
|
|
92
|
+
else
|
|
93
|
+
return 0
|
|
94
|
+
end
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
REQUEUE_SCRIPT = """
|
|
98
|
+
local removed = redis.call('ZREM', KEYS[1], ARGV[1])
|
|
99
|
+
if removed > 0 then
|
|
100
|
+
redis.call('LPUSH', KEYS[2], ARGV[1])
|
|
101
|
+
redis.call('DECR', KEYS[3])
|
|
102
|
+
redis.call('INCR', KEYS[4])
|
|
103
|
+
return 1
|
|
104
|
+
end
|
|
105
|
+
return 0
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
CRON_ENQUEUE_SCRIPT = """
|
|
109
|
+
local acquired = redis.call('SET', KEYS[1], '1', 'NX', 'EX', 2592000)
|
|
110
|
+
if acquired then
|
|
111
|
+
redis.call('LPUSH', KEYS[2], ARGV[1])
|
|
112
|
+
redis.call('INCR', KEYS[3])
|
|
113
|
+
return 1
|
|
114
|
+
end
|
|
115
|
+
return 0
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
PRIORITY_POP_SCRIPT = """
|
|
119
|
+
local dest = KEYS[1]
|
|
120
|
+
for i = 2, #KEYS do
|
|
121
|
+
local val = redis.call('RPOPLPUSH', KEYS[i], dest)
|
|
122
|
+
if val then
|
|
123
|
+
return val
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
return nil
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def is_leader() -> bool:
|
|
131
|
+
try:
|
|
132
|
+
acquired = await redis_client.r.set("pytaskq:leader_lock", WORKER_ID, nx=True, ex=20)
|
|
133
|
+
if acquired:
|
|
134
|
+
return True
|
|
135
|
+
|
|
136
|
+
current_leader = await redis_client.r.get("pytaskq:leader_lock")
|
|
137
|
+
if current_leader == WORKER_ID:
|
|
138
|
+
await redis_client.r.expire("pytaskq:leader_lock", 20)
|
|
139
|
+
return True
|
|
140
|
+
|
|
141
|
+
return False
|
|
142
|
+
except Exception as e:
|
|
143
|
+
logger.warning(f"[Leader Election] Redis error: {e}")
|
|
144
|
+
return False
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
async def cron_scheduler():
|
|
148
|
+
while True:
|
|
149
|
+
if not await is_leader():
|
|
150
|
+
await asyncio.sleep(10)
|
|
151
|
+
continue
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
now = datetime.now()
|
|
155
|
+
for task_name, task_info in list(TASKS.items()):
|
|
156
|
+
cron_expr = task_info.get("cron")
|
|
157
|
+
if not cron_expr:
|
|
158
|
+
continue
|
|
159
|
+
|
|
160
|
+
itr = croniter.croniter(cron_expr, now)
|
|
161
|
+
prev_tick = itr.get_prev(datetime)
|
|
162
|
+
tick_ts = int(prev_tick.timestamp())
|
|
163
|
+
|
|
164
|
+
if (now.timestamp() - tick_ts) < 60:
|
|
165
|
+
lock_key = f"cron_lock:{task_name}:{tick_ts}"
|
|
166
|
+
payload = queueing.new_payload(task_name, priority=task_info.get("priority") or "default")
|
|
167
|
+
enqueued = await redis_client.r.eval(
|
|
168
|
+
CRON_ENQUEUE_SCRIPT, 3,
|
|
169
|
+
lock_key, queueing.queue_key(payload["priority"]), "stats:pending",
|
|
170
|
+
json.dumps(payload),
|
|
171
|
+
)
|
|
172
|
+
if enqueued:
|
|
173
|
+
await queueing.record(redis_client.r, payload)
|
|
174
|
+
logger.info(f"[Cron] Scheduled {task_name} for tick {tick_ts}")
|
|
175
|
+
|
|
176
|
+
except Exception as e:
|
|
177
|
+
logger.error(f"[Cron Scheduler] Error evaluating cron schedules: {e}")
|
|
178
|
+
|
|
179
|
+
await asyncio.sleep(10)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
async def retry_scheduler():
|
|
183
|
+
while True:
|
|
184
|
+
if not await is_leader():
|
|
185
|
+
await asyncio.sleep(5)
|
|
186
|
+
continue
|
|
187
|
+
try:
|
|
188
|
+
await _promote_due_tasks()
|
|
189
|
+
except Exception as e:
|
|
190
|
+
logger.error(f"[Retry Scheduler] Redis error: {e}")
|
|
191
|
+
await asyncio.sleep(1)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
async def _promote_due_tasks(now: float = None) -> int:
|
|
195
|
+
"""Move delayed/retry tasks whose time has come onto their priority queue."""
|
|
196
|
+
now = time.time() if now is None else now
|
|
197
|
+
ready_tasks = await redis_client.r.zrangebyscore("delayed_tasks", "-inf", now)
|
|
198
|
+
moved = 0
|
|
199
|
+
for i, task_json in enumerate(ready_tasks):
|
|
200
|
+
if i > 0 and i % 100 == 0:
|
|
201
|
+
await redis_client.r.expire("pytaskq:leader_lock", 20)
|
|
202
|
+
try:
|
|
203
|
+
task_data = json.loads(task_json)
|
|
204
|
+
priority = task_data.get("priority", "default")
|
|
205
|
+
requeued = await redis_client.r.eval(
|
|
206
|
+
REQUEUE_SCRIPT, 4,
|
|
207
|
+
"delayed_tasks", queueing.queue_key(priority),
|
|
208
|
+
"stats:delayed", "stats:pending",
|
|
209
|
+
task_json,
|
|
210
|
+
)
|
|
211
|
+
if requeued:
|
|
212
|
+
moved += 1
|
|
213
|
+
task_id = task_data.get("task_id")
|
|
214
|
+
if task_id and task_data.get("retry_count", 0) == 0:
|
|
215
|
+
# A countdown task is now simply waiting in its queue
|
|
216
|
+
if await redis_client.r.hget(queueing.task_key(task_id), "status") == queueing.SCHEDULED:
|
|
217
|
+
await redis_client.r.hset(queueing.task_key(task_id), "status", queueing.QUEUED)
|
|
218
|
+
logger.info(f"[Scheduler] Queued task {task_id} (retry #{task_data.get('retry_count', 0)})")
|
|
219
|
+
except json.JSONDecodeError:
|
|
220
|
+
logger.error("[Scheduler] Malformed task found in delayed_tasks. Discarding.")
|
|
221
|
+
await redis_client.r.zrem("delayed_tasks", task_json)
|
|
222
|
+
except Exception as e:
|
|
223
|
+
logger.error(f"[Scheduler] Error processing task: {e}")
|
|
224
|
+
return moved
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _worker_info() -> dict:
|
|
228
|
+
return {
|
|
229
|
+
"worker_id": WORKER_ID,
|
|
230
|
+
"hostname": HOSTNAME,
|
|
231
|
+
"pid": os.getpid(),
|
|
232
|
+
"app": APP_NAME,
|
|
233
|
+
"cpu_workers": CPU_WORKERS,
|
|
234
|
+
"io_workers": IO_WORKERS,
|
|
235
|
+
"tasks": len([t for t in TASKS if not t.startswith("_")]),
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
async def _publish_registry():
|
|
240
|
+
from pytaskq.app import describe_tasks
|
|
241
|
+
if TASKS:
|
|
242
|
+
await queueing.publish_registry(redis_client.r, describe_tasks(TASKS))
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
async def heartbeat():
|
|
246
|
+
last_registry = 0.0
|
|
247
|
+
while True:
|
|
248
|
+
try:
|
|
249
|
+
now = time.time()
|
|
250
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
251
|
+
pipe.zadd("active_workers", {WORKER_ID: now + HEARTBEAT_TTL})
|
|
252
|
+
pipe.hset(f"worker:{WORKER_ID}", mapping={"last_seen": now, "active": len(active_tasks)})
|
|
253
|
+
pipe.expire(f"worker:{WORKER_ID}", HEARTBEAT_TTL * 4)
|
|
254
|
+
await pipe.execute()
|
|
255
|
+
if now - last_registry > 60:
|
|
256
|
+
await _publish_registry()
|
|
257
|
+
last_registry = now
|
|
258
|
+
await asyncio.sleep(10)
|
|
259
|
+
except Exception as e:
|
|
260
|
+
logger.warning(f"[Heartbeat] Redis unreachable: {e}. Retrying in 5s")
|
|
261
|
+
await asyncio.sleep(5)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
async def zombie_sweeper():
|
|
265
|
+
while True:
|
|
266
|
+
if not await is_leader():
|
|
267
|
+
await asyncio.sleep(5)
|
|
268
|
+
continue
|
|
269
|
+
try:
|
|
270
|
+
await _sweep_dead_workers()
|
|
271
|
+
except Exception as e:
|
|
272
|
+
logger.error(f"[Sweeper] Redis error: {e}. Retrying in 5s...")
|
|
273
|
+
await asyncio.sleep(5)
|
|
274
|
+
await asyncio.sleep(15)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
async def _sweep_dead_workers() -> int:
|
|
278
|
+
"""Re-queue in-flight tasks of workers whose heartbeat expired."""
|
|
279
|
+
expired = await redis_client.r.zrangebyscore("active_workers", "-inf", time.time())
|
|
280
|
+
total = 0
|
|
281
|
+
for worker_id in expired:
|
|
282
|
+
tasks_recovered = 0
|
|
283
|
+
while True:
|
|
284
|
+
tasks = await redis_client.r.lrange(f"processing_queue:{worker_id}", -1, -1)
|
|
285
|
+
if not tasks:
|
|
286
|
+
break
|
|
287
|
+
|
|
288
|
+
task = tasks[0]
|
|
289
|
+
if tasks_recovered > 0 and tasks_recovered % 100 == 0:
|
|
290
|
+
await redis_client.r.expire("pytaskq:leader_lock", 20)
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
task_data = json.loads(task)
|
|
294
|
+
task_data["fence_token"] = int(task_data.get("fence_token", 0)) + 1
|
|
295
|
+
task_id = task_data.get("task_id")
|
|
296
|
+
new_raw = json.dumps(task_data)
|
|
297
|
+
|
|
298
|
+
async with redis_client.r.pipeline(transaction=True) as pipe:
|
|
299
|
+
pipe.lrem(f"processing_queue:{worker_id}", count=1, value=task)
|
|
300
|
+
pipe.set(f"fence:{task_id}", task_data["fence_token"], ex=86400)
|
|
301
|
+
pipe.lpush("queue:high", new_raw)
|
|
302
|
+
pipe.incr("stats:pending")
|
|
303
|
+
pipe.decr("stats:processing")
|
|
304
|
+
if task_id and await redis_client.r.exists(queueing.task_key(task_id)):
|
|
305
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
306
|
+
"status": queueing.QUEUED, "priority": "high", "payload": new_raw,
|
|
307
|
+
"note": f"Recovered from dead worker {worker_id}",
|
|
308
|
+
})
|
|
309
|
+
await pipe.execute()
|
|
310
|
+
except json.JSONDecodeError:
|
|
311
|
+
logger.error(f"[Sweeper] Malformed task found in {worker_id} processing queue. Discarding.")
|
|
312
|
+
await redis_client.r.lrem(f"processing_queue:{worker_id}", count=1, value=task)
|
|
313
|
+
|
|
314
|
+
tasks_recovered += 1
|
|
315
|
+
|
|
316
|
+
await redis_client.r.zrem("active_workers", worker_id)
|
|
317
|
+
await redis_client.r.delete(f"worker:{worker_id}")
|
|
318
|
+
total += tasks_recovered
|
|
319
|
+
logger.info(f"[Sweeper] Recovered {tasks_recovered} task(s) from dead worker: {worker_id}")
|
|
320
|
+
return total
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _resolve(value, entry, key, default):
|
|
324
|
+
if value is not None:
|
|
325
|
+
return value
|
|
326
|
+
if entry and entry.get(key) is not None:
|
|
327
|
+
return entry[key]
|
|
328
|
+
return default
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
async def handle_task(task_json, sem, loop, process_pool, thread_pool):
|
|
332
|
+
incr_done = False
|
|
333
|
+
r = redis_client.r
|
|
334
|
+
try:
|
|
335
|
+
# Immediately decrement pending since we pulled it from the queue
|
|
336
|
+
await r.decr("stats:pending")
|
|
337
|
+
|
|
338
|
+
try:
|
|
339
|
+
tasks = Taskloader.model_validate_json(task_json)
|
|
340
|
+
task_id = tasks.task_id
|
|
341
|
+
my_token = tasks.fence_token
|
|
342
|
+
except Exception as e:
|
|
343
|
+
logger.error(f"Task validation error: {e}")
|
|
344
|
+
try:
|
|
345
|
+
task_id = str(json.loads(task_json).get("task_id") or "Unknown")
|
|
346
|
+
except Exception:
|
|
347
|
+
task_id = "Unknown"
|
|
348
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
349
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
350
|
+
"task_id": task_id,
|
|
351
|
+
"status": queueing.FAILED,
|
|
352
|
+
"error": f"Invalid task payload: {e}",
|
|
353
|
+
"finished_at": time.time(),
|
|
354
|
+
})
|
|
355
|
+
pipe.incr("stats:failed")
|
|
356
|
+
await pipe.execute()
|
|
357
|
+
return
|
|
358
|
+
|
|
359
|
+
entry = TASKS.get(tasks.task_name)
|
|
360
|
+
max_retries = int(_resolve(tasks.max_retries, entry, "max_retries", settings.MAX_RETRIES))
|
|
361
|
+
timeout = float(_resolve(tasks.timeout, entry, "timeout", TASK_TIMEOUT))
|
|
362
|
+
|
|
363
|
+
parent_ctx = extract_trace_context(getattr(tasks, "trace_carrier", {}))
|
|
364
|
+
with tracer.start_as_current_span(
|
|
365
|
+
"execute_task",
|
|
366
|
+
context=parent_ctx,
|
|
367
|
+
attributes={
|
|
368
|
+
"task.id": task_id,
|
|
369
|
+
"task.name": tasks.task_name,
|
|
370
|
+
"task.retry_count": tasks.retry_count,
|
|
371
|
+
"worker.id": WORKER_ID,
|
|
372
|
+
},
|
|
373
|
+
) as span:
|
|
374
|
+
started_at = time.time()
|
|
375
|
+
try:
|
|
376
|
+
if entry is None:
|
|
377
|
+
known = ", ".join(sorted(t for t in TASKS if not t.startswith("_"))) or "none"
|
|
378
|
+
raise UnknownTaskError(
|
|
379
|
+
f"Task '{tasks.task_name}' is not registered on worker {WORKER_ID}. "
|
|
380
|
+
f"Registered tasks: {known}"
|
|
381
|
+
)
|
|
382
|
+
func = entry["handler"]
|
|
383
|
+
task_type = entry.get("type", "cpu")
|
|
384
|
+
|
|
385
|
+
logger.info(f"Executing task {task_id} ({tasks.task_name})")
|
|
386
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
387
|
+
pipe.incr("stats:processing")
|
|
388
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
389
|
+
"task_id": task_id,
|
|
390
|
+
"task_name": tasks.task_name,
|
|
391
|
+
"status": queueing.PROCESSING,
|
|
392
|
+
"started_at": started_at,
|
|
393
|
+
"worker_id": WORKER_ID,
|
|
394
|
+
"retry_count": tasks.retry_count,
|
|
395
|
+
})
|
|
396
|
+
pipe.publish(queueing.EVENTS_CHANNEL, "update")
|
|
397
|
+
await pipe.execute()
|
|
398
|
+
incr_done = True
|
|
399
|
+
|
|
400
|
+
args, kwargs = tasks.args, tasks.kwargs
|
|
401
|
+
if inspect.iscoroutinefunction(func):
|
|
402
|
+
exec_future = asyncio.ensure_future(func(*args, **kwargs))
|
|
403
|
+
else:
|
|
404
|
+
call = functools.partial(func, *args, **kwargs) if kwargs else func
|
|
405
|
+
call_args = () if kwargs else tuple(args)
|
|
406
|
+
pool = process_pool if (task_type == "cpu" and process_pool is not None) else thread_pool
|
|
407
|
+
exec_future = loop.run_in_executor(pool, call, *call_args)
|
|
408
|
+
|
|
409
|
+
if timeout > 0:
|
|
410
|
+
try:
|
|
411
|
+
result = await asyncio.wait_for(exec_future, timeout=timeout)
|
|
412
|
+
except asyncio.TimeoutError:
|
|
413
|
+
# Invalidate this attempt; the retry carries the new token so
|
|
414
|
+
# ITS result is accepted while any late write from this one is not.
|
|
415
|
+
new_token = await r.incr(f"fence:{task_id}")
|
|
416
|
+
await r.expire(f"fence:{task_id}", 86400)
|
|
417
|
+
tasks.fence_token = int(new_token)
|
|
418
|
+
raise TaskTimeoutError(
|
|
419
|
+
f"Task exceeded {timeout:g}s limit. Execution abandoned."
|
|
420
|
+
) from None
|
|
421
|
+
else:
|
|
422
|
+
result = await exec_future
|
|
423
|
+
|
|
424
|
+
saved = await r.eval(
|
|
425
|
+
FENCE_SCRIPT, 2,
|
|
426
|
+
f"fence:{task_id}", queueing.task_key(task_id),
|
|
427
|
+
str(my_token), task_id, queueing.SUCCESS, queueing.encode_result(result),
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
if saved:
|
|
431
|
+
finished_at = time.time()
|
|
432
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
433
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
434
|
+
"finished_at": finished_at,
|
|
435
|
+
"duration_ms": round((finished_at - started_at) * 1000, 1),
|
|
436
|
+
"error": "",
|
|
437
|
+
})
|
|
438
|
+
pipe.expire(queueing.task_key(task_id), settings.RESULT_TTL)
|
|
439
|
+
pipe.incr("stats:completed")
|
|
440
|
+
pipe.hincrby(f"worker:{WORKER_ID}", "processed", 1)
|
|
441
|
+
await pipe.execute()
|
|
442
|
+
span.set_attribute("task.status", "Success")
|
|
443
|
+
logger.info(f"Task {task_id} completed successfully.")
|
|
444
|
+
# ── Webhook delivery ──────────────────────────────────────
|
|
445
|
+
# Travels through the queue like any other task, so network
|
|
446
|
+
# blips are retried automatically.
|
|
447
|
+
if tasks.on_success_url:
|
|
448
|
+
webhook = queueing.new_payload(
|
|
449
|
+
"_deliver_webhook",
|
|
450
|
+
[tasks.on_success_url, task_id, tasks.task_name, queueing.encode_result(result)],
|
|
451
|
+
)
|
|
452
|
+
await queueing.enqueue(r, webhook)
|
|
453
|
+
logger.info(f"Enqueued webhook delivery for task {task_id} → {tasks.on_success_url}")
|
|
454
|
+
else:
|
|
455
|
+
span.set_attribute("task.status", "Discarded")
|
|
456
|
+
span.set_attribute("task.discard_reason", "fence_token_stale")
|
|
457
|
+
logger.warning(f"Task {task_id} discarded (stale fence token).")
|
|
458
|
+
|
|
459
|
+
except Exception as e:
|
|
460
|
+
error = f"{type(e).__name__}: {e}"
|
|
461
|
+
logger.error(f"Task {task_id} failed: {error}")
|
|
462
|
+
span.set_status(trace.StatusCode.ERROR, str(e))
|
|
463
|
+
span.record_exception(e)
|
|
464
|
+
finished_at = time.time()
|
|
465
|
+
await r.incr("stats:failed")
|
|
466
|
+
await r.hincrby(f"worker:{WORKER_ID}", "failed", 1)
|
|
467
|
+
|
|
468
|
+
if tasks.retry_count >= max_retries:
|
|
469
|
+
logger.error(f"[DLQ] Task {task_id} failed after {tasks.retry_count} retries. Moving to DLQ.")
|
|
470
|
+
span.set_attribute("task.status", "DeadLetter")
|
|
471
|
+
dlq_entry = json.loads(task_json)
|
|
472
|
+
dlq_entry["fence_token"] = tasks.fence_token
|
|
473
|
+
dlq_entry["error"] = error
|
|
474
|
+
dlq_entry["failed_at"] = finished_at
|
|
475
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
476
|
+
pipe.lpush("dlq", json.dumps(dlq_entry))
|
|
477
|
+
pipe.incr("stats:dlq")
|
|
478
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
479
|
+
"task_id": task_id,
|
|
480
|
+
"status": queueing.DEAD_LETTER,
|
|
481
|
+
"finished_at": finished_at,
|
|
482
|
+
"error": f"Failed after {tasks.retry_count} retries. Last error: {error}",
|
|
483
|
+
})
|
|
484
|
+
await pipe.execute()
|
|
485
|
+
else:
|
|
486
|
+
tasks.retry_count += 1
|
|
487
|
+
BASE = 1.0
|
|
488
|
+
MAX_DELAY = 30.0
|
|
489
|
+
prev = getattr(tasks, "prev_delay", BASE)
|
|
490
|
+
delay = min(MAX_DELAY, random.uniform(BASE, prev * 3))
|
|
491
|
+
tasks.prev_delay = delay
|
|
492
|
+
retry_raw = json.dumps(tasks.model_dump())
|
|
493
|
+
|
|
494
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
495
|
+
pipe.zadd("delayed_tasks", {retry_raw: time.time() + delay})
|
|
496
|
+
pipe.incr("stats:delayed")
|
|
497
|
+
pipe.hset(queueing.task_key(task_id), mapping={
|
|
498
|
+
"task_id": task_id,
|
|
499
|
+
"status": queueing.RETRY_SCHEDULED,
|
|
500
|
+
"retry_count": tasks.retry_count,
|
|
501
|
+
"next_retry_at": time.time() + delay,
|
|
502
|
+
"payload": retry_raw,
|
|
503
|
+
"error": f"{error}. Retry {tasks.retry_count}/{max_retries} in {delay:.1f}s.",
|
|
504
|
+
})
|
|
505
|
+
await pipe.execute()
|
|
506
|
+
|
|
507
|
+
finally:
|
|
508
|
+
try:
|
|
509
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
510
|
+
pipe.lrem(f"processing_queue:{WORKER_ID}", count=1, value=task_json)
|
|
511
|
+
if incr_done:
|
|
512
|
+
pipe.decr("stats:processing")
|
|
513
|
+
pipe.publish(queueing.EVENTS_CHANNEL, "update")
|
|
514
|
+
await pipe.execute()
|
|
515
|
+
except Exception as e:
|
|
516
|
+
logger.error(f"[Cleanup] Redis error: {e}")
|
|
517
|
+
finally:
|
|
518
|
+
sem.release()
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
async def _recover_own_tasks():
|
|
522
|
+
"""On boot, put back anything this worker id left in its processing queue."""
|
|
523
|
+
while True:
|
|
524
|
+
leftover = await redis_client.r.rpoplpush(f"processing_queue:{WORKER_ID}", "queue:high")
|
|
525
|
+
if not leftover:
|
|
526
|
+
break
|
|
527
|
+
await redis_client.r.incr("stats:pending")
|
|
528
|
+
await redis_client.r.decr("stats:processing")
|
|
529
|
+
logger.info(f"Recovered crashed task on startup: {leftover}")
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
async def consumer_task():
|
|
533
|
+
concurrency = CPU_WORKERS + IO_WORKERS
|
|
534
|
+
logger.info(
|
|
535
|
+
f"Worker {WORKER_ID}: {_TOTAL_CORES} CPU cores detected. "
|
|
536
|
+
f"CPU processes={CPU_WORKERS}, IO threads={IO_WORKERS}, max concurrent tasks={concurrency}"
|
|
537
|
+
)
|
|
538
|
+
user_tasks = sorted(t for t in TASKS if not t.startswith("_"))
|
|
539
|
+
logger.info(f"Registered tasks ({len(user_tasks)}): {', '.join(user_tasks) or 'none'}")
|
|
540
|
+
|
|
541
|
+
process_pool = ProcessPoolExecutor(max_workers=CPU_WORKERS)
|
|
542
|
+
thread_pool = ThreadPoolExecutor(max_workers=IO_WORKERS)
|
|
543
|
+
loop = asyncio.get_running_loop()
|
|
544
|
+
sem = asyncio.Semaphore(concurrency)
|
|
545
|
+
|
|
546
|
+
def _signal_handler():
|
|
547
|
+
logger.info("Shutdown signal received. Finishing running tasks...")
|
|
548
|
+
shutdown_event.set()
|
|
549
|
+
|
|
550
|
+
try:
|
|
551
|
+
loop.add_signal_handler(signal.SIGINT, _signal_handler)
|
|
552
|
+
loop.add_signal_handler(signal.SIGTERM, _signal_handler)
|
|
553
|
+
except NotImplementedError: # Windows
|
|
554
|
+
signal.signal(signal.SIGINT, lambda s, f: loop.call_soon_threadsafe(_signal_handler))
|
|
555
|
+
signal.signal(signal.SIGTERM, lambda s, f: loop.call_soon_threadsafe(_signal_handler))
|
|
556
|
+
|
|
557
|
+
while not shutdown_event.is_set():
|
|
558
|
+
try:
|
|
559
|
+
await _recover_own_tasks()
|
|
560
|
+
await redis_client.r.hset(f"worker:{WORKER_ID}", mapping={**_worker_info(), "started_at": time.time()})
|
|
561
|
+
await _publish_registry()
|
|
562
|
+
break
|
|
563
|
+
except Exception as e:
|
|
564
|
+
logger.error(f"[Startup] Redis unavailable ({e.__class__.__name__}). Retrying in 5s...")
|
|
565
|
+
await asyncio.sleep(5)
|
|
566
|
+
|
|
567
|
+
background = [
|
|
568
|
+
asyncio.create_task(cron_scheduler()),
|
|
569
|
+
asyncio.create_task(retry_scheduler()),
|
|
570
|
+
asyncio.create_task(heartbeat()),
|
|
571
|
+
asyncio.create_task(zombie_sweeper()),
|
|
572
|
+
]
|
|
573
|
+
|
|
574
|
+
logger.info(f"Worker {WORKER_ID} started and ready to process tasks.")
|
|
575
|
+
|
|
576
|
+
idle_delay = 0.05
|
|
577
|
+
max_idle_delay = 1.0
|
|
578
|
+
|
|
579
|
+
while not shutdown_event.is_set():
|
|
580
|
+
try:
|
|
581
|
+
await sem.acquire()
|
|
582
|
+
task_json = await redis_client.r.eval(
|
|
583
|
+
PRIORITY_POP_SCRIPT,
|
|
584
|
+
4,
|
|
585
|
+
f"processing_queue:{WORKER_ID}",
|
|
586
|
+
"queue:high",
|
|
587
|
+
"queue:default",
|
|
588
|
+
"queue:low",
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
if not task_json:
|
|
592
|
+
sem.release()
|
|
593
|
+
try:
|
|
594
|
+
await asyncio.wait_for(shutdown_event.wait(), timeout=idle_delay)
|
|
595
|
+
except asyncio.TimeoutError:
|
|
596
|
+
pass
|
|
597
|
+
idle_delay = min(idle_delay * 1.5, max_idle_delay)
|
|
598
|
+
continue
|
|
599
|
+
|
|
600
|
+
idle_delay = 0.05
|
|
601
|
+
except Exception as e:
|
|
602
|
+
sem.release()
|
|
603
|
+
logger.error(f"[Main loop] Redis error: {e.__class__.__name__}. Backing off 5s...")
|
|
604
|
+
await asyncio.sleep(5)
|
|
605
|
+
continue
|
|
606
|
+
|
|
607
|
+
task = asyncio.create_task(
|
|
608
|
+
handle_task(task_json, sem, loop, process_pool, thread_pool)
|
|
609
|
+
)
|
|
610
|
+
active_tasks.add(task)
|
|
611
|
+
task.add_done_callback(active_tasks.discard)
|
|
612
|
+
|
|
613
|
+
if active_tasks:
|
|
614
|
+
logger.info(f"Waiting for {len(active_tasks)} active tasks to complete...")
|
|
615
|
+
await asyncio.gather(*active_tasks, return_exceptions=True)
|
|
616
|
+
|
|
617
|
+
for bg in background:
|
|
618
|
+
bg.cancel()
|
|
619
|
+
try:
|
|
620
|
+
await redis_client.r.zrem("active_workers", WORKER_ID)
|
|
621
|
+
await redis_client.r.delete(f"worker:{WORKER_ID}")
|
|
622
|
+
except Exception:
|
|
623
|
+
pass
|
|
624
|
+
|
|
625
|
+
thread_pool.shutdown(wait=True)
|
|
626
|
+
process_pool.shutdown(wait=True)
|
|
627
|
+
await redis_client.r.aclose()
|
|
628
|
+
|
|
629
|
+
provider = trace.get_tracer_provider()
|
|
630
|
+
if hasattr(provider, "force_flush"):
|
|
631
|
+
provider.force_flush()
|
|
632
|
+
logger.info(f"Worker {WORKER_ID} stopped cleanly.")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
if __name__ == "__main__":
|
|
636
|
+
asyncio.run(consumer_task())
|