pytaskq 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pytaskq/worker.py ADDED
@@ -0,0 +1,636 @@
1
+ """
2
+ pytaskq.worker — the task-executing process
3
+ ============================================
4
+
5
+ Start one (or many, on many machines) with:
6
+
7
+ pytaskq worker --app myproject.tasks:queue
8
+
9
+ Each worker pops tasks in strict priority order (high → default → low), runs
10
+ them in a process pool (type="cpu"), a thread pool (type="io") or directly on
11
+ the event loop (async def), and records status, timings and results in Redis.
12
+ Background loops (only on the elected leader) handle cron schedules, delayed
13
+ retries and recovery of tasks from crashed workers.
14
+ """
15
+
16
+ import asyncio
17
+ import functools
18
+ import inspect
19
+ import json
20
+ import logging
21
+ import os
22
+ import random
23
+ import signal
24
+ import socket
25
+ import time
26
+ import uuid
27
+ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
28
+ from datetime import datetime
29
+
30
+ import croniter
31
+ from opentelemetry import trace
32
+
33
+ from pytaskq.core import queueing
34
+ from pytaskq.core import redis_client
35
+ from pytaskq.core.config import settings
36
+ from pytaskq.schemas.requests import Taskloader, Taskresult # noqa: F401 (Taskresult kept for API compat)
37
+ from pytaskq.tracing import extract_trace_context, init_tracer, inject_trace_context # noqa: F401
38
+
39
+ TASKS = {}
40
+ APP_NAME = ""
41
+
42
+
43
+ def set_tasks(tasks_dict, app_name: str = ""):
44
+ """Register the tasks this worker may execute (called by the CLI)."""
45
+ global APP_NAME
46
+ TASKS.update(tasks_dict)
47
+ if "_deliver_webhook" not in TASKS:
48
+ # Webhooks must work for every app, even one built from a plain dict.
49
+ from pytaskq.app import _deliver_webhook
50
+ TASKS["_deliver_webhook"] = {"handler": _deliver_webhook, "type": "io"}
51
+ if app_name:
52
+ APP_NAME = app_name
53
+
54
+
55
+ logging.basicConfig(
56
+ level=logging.INFO,
57
+ format="%(asctime)s [%(levelname)s] [%(processName)s] %(message)s",
58
+ datefmt="%Y-%m-%d %H:%M:%S",
59
+ )
60
+ logger = logging.getLogger("worker")
61
+
62
+ HOSTNAME = socket.gethostname()
63
+ WORKER_ID = f"{HOSTNAME}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
64
+ TASK_TIMEOUT = float(os.getenv("TASK_TIMEOUT", str(settings.TASK_TIMEOUT)))
65
+ HEARTBEAT_TTL = 30
66
+
67
+ _TOTAL_CORES = os.cpu_count() or 1
68
+ _RESERVED_CORES = int(os.getenv("WORKER_RESERVED_CORES", "1"))
69
+ CPU_WORKERS = int(os.getenv("WORKER_CPU_WORKERS", "0")) or max(1, _TOTAL_CORES - _RESERVED_CORES)
70
+ IO_WORKERS = int(os.getenv("WORKER_IO_WORKERS", "0")) or max(4, _TOTAL_CORES * 4)
71
+ CONCURRENCY = CPU_WORKERS + IO_WORKERS
72
+
73
+
74
+ class TaskTimeoutError(Exception):
75
+ """Raised when a task exceeds its timeout."""
76
+
77
+
78
+ class UnknownTaskError(Exception):
79
+ """Raised when a payload names a task this worker does not know."""
80
+
81
+
82
+ shutdown_event = asyncio.Event()
83
+ active_tasks = set()
84
+ tracer = init_tracer("pytaskq-worker")
85
+
86
+ FENCE_SCRIPT = """
87
+ local current = redis.call('get', KEYS[1])
88
+ if current == false or current == ARGV[1] then
89
+ redis.call('hset', KEYS[2], 'task_id', ARGV[2], 'status', ARGV[3], 'result', ARGV[4])
90
+ redis.call('expire', KEYS[2], 86400)
91
+ return 1
92
+ else
93
+ return 0
94
+ end
95
+ """
96
+
97
+ REQUEUE_SCRIPT = """
98
+ local removed = redis.call('ZREM', KEYS[1], ARGV[1])
99
+ if removed > 0 then
100
+ redis.call('LPUSH', KEYS[2], ARGV[1])
101
+ redis.call('DECR', KEYS[3])
102
+ redis.call('INCR', KEYS[4])
103
+ return 1
104
+ end
105
+ return 0
106
+ """
107
+
108
+ CRON_ENQUEUE_SCRIPT = """
109
+ local acquired = redis.call('SET', KEYS[1], '1', 'NX', 'EX', 2592000)
110
+ if acquired then
111
+ redis.call('LPUSH', KEYS[2], ARGV[1])
112
+ redis.call('INCR', KEYS[3])
113
+ return 1
114
+ end
115
+ return 0
116
+ """
117
+
118
+ PRIORITY_POP_SCRIPT = """
119
+ local dest = KEYS[1]
120
+ for i = 2, #KEYS do
121
+ local val = redis.call('RPOPLPUSH', KEYS[i], dest)
122
+ if val then
123
+ return val
124
+ end
125
+ end
126
+ return nil
127
+ """
128
+
129
+
130
+ async def is_leader() -> bool:
131
+ try:
132
+ acquired = await redis_client.r.set("pytaskq:leader_lock", WORKER_ID, nx=True, ex=20)
133
+ if acquired:
134
+ return True
135
+
136
+ current_leader = await redis_client.r.get("pytaskq:leader_lock")
137
+ if current_leader == WORKER_ID:
138
+ await redis_client.r.expire("pytaskq:leader_lock", 20)
139
+ return True
140
+
141
+ return False
142
+ except Exception as e:
143
+ logger.warning(f"[Leader Election] Redis error: {e}")
144
+ return False
145
+
146
+
147
+ async def cron_scheduler():
148
+ while True:
149
+ if not await is_leader():
150
+ await asyncio.sleep(10)
151
+ continue
152
+
153
+ try:
154
+ now = datetime.now()
155
+ for task_name, task_info in list(TASKS.items()):
156
+ cron_expr = task_info.get("cron")
157
+ if not cron_expr:
158
+ continue
159
+
160
+ itr = croniter.croniter(cron_expr, now)
161
+ prev_tick = itr.get_prev(datetime)
162
+ tick_ts = int(prev_tick.timestamp())
163
+
164
+ if (now.timestamp() - tick_ts) < 60:
165
+ lock_key = f"cron_lock:{task_name}:{tick_ts}"
166
+ payload = queueing.new_payload(task_name, priority=task_info.get("priority") or "default")
167
+ enqueued = await redis_client.r.eval(
168
+ CRON_ENQUEUE_SCRIPT, 3,
169
+ lock_key, queueing.queue_key(payload["priority"]), "stats:pending",
170
+ json.dumps(payload),
171
+ )
172
+ if enqueued:
173
+ await queueing.record(redis_client.r, payload)
174
+ logger.info(f"[Cron] Scheduled {task_name} for tick {tick_ts}")
175
+
176
+ except Exception as e:
177
+ logger.error(f"[Cron Scheduler] Error evaluating cron schedules: {e}")
178
+
179
+ await asyncio.sleep(10)
180
+
181
+
182
+ async def retry_scheduler():
183
+ while True:
184
+ if not await is_leader():
185
+ await asyncio.sleep(5)
186
+ continue
187
+ try:
188
+ await _promote_due_tasks()
189
+ except Exception as e:
190
+ logger.error(f"[Retry Scheduler] Redis error: {e}")
191
+ await asyncio.sleep(1)
192
+
193
+
194
+ async def _promote_due_tasks(now: float = None) -> int:
195
+ """Move delayed/retry tasks whose time has come onto their priority queue."""
196
+ now = time.time() if now is None else now
197
+ ready_tasks = await redis_client.r.zrangebyscore("delayed_tasks", "-inf", now)
198
+ moved = 0
199
+ for i, task_json in enumerate(ready_tasks):
200
+ if i > 0 and i % 100 == 0:
201
+ await redis_client.r.expire("pytaskq:leader_lock", 20)
202
+ try:
203
+ task_data = json.loads(task_json)
204
+ priority = task_data.get("priority", "default")
205
+ requeued = await redis_client.r.eval(
206
+ REQUEUE_SCRIPT, 4,
207
+ "delayed_tasks", queueing.queue_key(priority),
208
+ "stats:delayed", "stats:pending",
209
+ task_json,
210
+ )
211
+ if requeued:
212
+ moved += 1
213
+ task_id = task_data.get("task_id")
214
+ if task_id and task_data.get("retry_count", 0) == 0:
215
+ # A countdown task is now simply waiting in its queue
216
+ if await redis_client.r.hget(queueing.task_key(task_id), "status") == queueing.SCHEDULED:
217
+ await redis_client.r.hset(queueing.task_key(task_id), "status", queueing.QUEUED)
218
+ logger.info(f"[Scheduler] Queued task {task_id} (retry #{task_data.get('retry_count', 0)})")
219
+ except json.JSONDecodeError:
220
+ logger.error("[Scheduler] Malformed task found in delayed_tasks. Discarding.")
221
+ await redis_client.r.zrem("delayed_tasks", task_json)
222
+ except Exception as e:
223
+ logger.error(f"[Scheduler] Error processing task: {e}")
224
+ return moved
225
+
226
+
227
+ def _worker_info() -> dict:
228
+ return {
229
+ "worker_id": WORKER_ID,
230
+ "hostname": HOSTNAME,
231
+ "pid": os.getpid(),
232
+ "app": APP_NAME,
233
+ "cpu_workers": CPU_WORKERS,
234
+ "io_workers": IO_WORKERS,
235
+ "tasks": len([t for t in TASKS if not t.startswith("_")]),
236
+ }
237
+
238
+
239
+ async def _publish_registry():
240
+ from pytaskq.app import describe_tasks
241
+ if TASKS:
242
+ await queueing.publish_registry(redis_client.r, describe_tasks(TASKS))
243
+
244
+
245
+ async def heartbeat():
246
+ last_registry = 0.0
247
+ while True:
248
+ try:
249
+ now = time.time()
250
+ async with redis_client.r.pipeline(transaction=True) as pipe:
251
+ pipe.zadd("active_workers", {WORKER_ID: now + HEARTBEAT_TTL})
252
+ pipe.hset(f"worker:{WORKER_ID}", mapping={"last_seen": now, "active": len(active_tasks)})
253
+ pipe.expire(f"worker:{WORKER_ID}", HEARTBEAT_TTL * 4)
254
+ await pipe.execute()
255
+ if now - last_registry > 60:
256
+ await _publish_registry()
257
+ last_registry = now
258
+ await asyncio.sleep(10)
259
+ except Exception as e:
260
+ logger.warning(f"[Heartbeat] Redis unreachable: {e}. Retrying in 5s")
261
+ await asyncio.sleep(5)
262
+
263
+
264
+ async def zombie_sweeper():
265
+ while True:
266
+ if not await is_leader():
267
+ await asyncio.sleep(5)
268
+ continue
269
+ try:
270
+ await _sweep_dead_workers()
271
+ except Exception as e:
272
+ logger.error(f"[Sweeper] Redis error: {e}. Retrying in 5s...")
273
+ await asyncio.sleep(5)
274
+ await asyncio.sleep(15)
275
+
276
+
277
+ async def _sweep_dead_workers() -> int:
278
+ """Re-queue in-flight tasks of workers whose heartbeat expired."""
279
+ expired = await redis_client.r.zrangebyscore("active_workers", "-inf", time.time())
280
+ total = 0
281
+ for worker_id in expired:
282
+ tasks_recovered = 0
283
+ while True:
284
+ tasks = await redis_client.r.lrange(f"processing_queue:{worker_id}", -1, -1)
285
+ if not tasks:
286
+ break
287
+
288
+ task = tasks[0]
289
+ if tasks_recovered > 0 and tasks_recovered % 100 == 0:
290
+ await redis_client.r.expire("pytaskq:leader_lock", 20)
291
+
292
+ try:
293
+ task_data = json.loads(task)
294
+ task_data["fence_token"] = int(task_data.get("fence_token", 0)) + 1
295
+ task_id = task_data.get("task_id")
296
+ new_raw = json.dumps(task_data)
297
+
298
+ async with redis_client.r.pipeline(transaction=True) as pipe:
299
+ pipe.lrem(f"processing_queue:{worker_id}", count=1, value=task)
300
+ pipe.set(f"fence:{task_id}", task_data["fence_token"], ex=86400)
301
+ pipe.lpush("queue:high", new_raw)
302
+ pipe.incr("stats:pending")
303
+ pipe.decr("stats:processing")
304
+ if task_id and await redis_client.r.exists(queueing.task_key(task_id)):
305
+ pipe.hset(queueing.task_key(task_id), mapping={
306
+ "status": queueing.QUEUED, "priority": "high", "payload": new_raw,
307
+ "note": f"Recovered from dead worker {worker_id}",
308
+ })
309
+ await pipe.execute()
310
+ except json.JSONDecodeError:
311
+ logger.error(f"[Sweeper] Malformed task found in {worker_id} processing queue. Discarding.")
312
+ await redis_client.r.lrem(f"processing_queue:{worker_id}", count=1, value=task)
313
+
314
+ tasks_recovered += 1
315
+
316
+ await redis_client.r.zrem("active_workers", worker_id)
317
+ await redis_client.r.delete(f"worker:{worker_id}")
318
+ total += tasks_recovered
319
+ logger.info(f"[Sweeper] Recovered {tasks_recovered} task(s) from dead worker: {worker_id}")
320
+ return total
321
+
322
+
323
+ def _resolve(value, entry, key, default):
324
+ if value is not None:
325
+ return value
326
+ if entry and entry.get(key) is not None:
327
+ return entry[key]
328
+ return default
329
+
330
+
331
+ async def handle_task(task_json, sem, loop, process_pool, thread_pool):
332
+ incr_done = False
333
+ r = redis_client.r
334
+ try:
335
+ # Immediately decrement pending since we pulled it from the queue
336
+ await r.decr("stats:pending")
337
+
338
+ try:
339
+ tasks = Taskloader.model_validate_json(task_json)
340
+ task_id = tasks.task_id
341
+ my_token = tasks.fence_token
342
+ except Exception as e:
343
+ logger.error(f"Task validation error: {e}")
344
+ try:
345
+ task_id = str(json.loads(task_json).get("task_id") or "Unknown")
346
+ except Exception:
347
+ task_id = "Unknown"
348
+ async with r.pipeline(transaction=True) as pipe:
349
+ pipe.hset(queueing.task_key(task_id), mapping={
350
+ "task_id": task_id,
351
+ "status": queueing.FAILED,
352
+ "error": f"Invalid task payload: {e}",
353
+ "finished_at": time.time(),
354
+ })
355
+ pipe.incr("stats:failed")
356
+ await pipe.execute()
357
+ return
358
+
359
+ entry = TASKS.get(tasks.task_name)
360
+ max_retries = int(_resolve(tasks.max_retries, entry, "max_retries", settings.MAX_RETRIES))
361
+ timeout = float(_resolve(tasks.timeout, entry, "timeout", TASK_TIMEOUT))
362
+
363
+ parent_ctx = extract_trace_context(getattr(tasks, "trace_carrier", {}))
364
+ with tracer.start_as_current_span(
365
+ "execute_task",
366
+ context=parent_ctx,
367
+ attributes={
368
+ "task.id": task_id,
369
+ "task.name": tasks.task_name,
370
+ "task.retry_count": tasks.retry_count,
371
+ "worker.id": WORKER_ID,
372
+ },
373
+ ) as span:
374
+ started_at = time.time()
375
+ try:
376
+ if entry is None:
377
+ known = ", ".join(sorted(t for t in TASKS if not t.startswith("_"))) or "none"
378
+ raise UnknownTaskError(
379
+ f"Task '{tasks.task_name}' is not registered on worker {WORKER_ID}. "
380
+ f"Registered tasks: {known}"
381
+ )
382
+ func = entry["handler"]
383
+ task_type = entry.get("type", "cpu")
384
+
385
+ logger.info(f"Executing task {task_id} ({tasks.task_name})")
386
+ async with r.pipeline(transaction=True) as pipe:
387
+ pipe.incr("stats:processing")
388
+ pipe.hset(queueing.task_key(task_id), mapping={
389
+ "task_id": task_id,
390
+ "task_name": tasks.task_name,
391
+ "status": queueing.PROCESSING,
392
+ "started_at": started_at,
393
+ "worker_id": WORKER_ID,
394
+ "retry_count": tasks.retry_count,
395
+ })
396
+ pipe.publish(queueing.EVENTS_CHANNEL, "update")
397
+ await pipe.execute()
398
+ incr_done = True
399
+
400
+ args, kwargs = tasks.args, tasks.kwargs
401
+ if inspect.iscoroutinefunction(func):
402
+ exec_future = asyncio.ensure_future(func(*args, **kwargs))
403
+ else:
404
+ call = functools.partial(func, *args, **kwargs) if kwargs else func
405
+ call_args = () if kwargs else tuple(args)
406
+ pool = process_pool if (task_type == "cpu" and process_pool is not None) else thread_pool
407
+ exec_future = loop.run_in_executor(pool, call, *call_args)
408
+
409
+ if timeout > 0:
410
+ try:
411
+ result = await asyncio.wait_for(exec_future, timeout=timeout)
412
+ except asyncio.TimeoutError:
413
+ # Invalidate this attempt; the retry carries the new token so
414
+ # ITS result is accepted while any late write from this one is not.
415
+ new_token = await r.incr(f"fence:{task_id}")
416
+ await r.expire(f"fence:{task_id}", 86400)
417
+ tasks.fence_token = int(new_token)
418
+ raise TaskTimeoutError(
419
+ f"Task exceeded {timeout:g}s limit. Execution abandoned."
420
+ ) from None
421
+ else:
422
+ result = await exec_future
423
+
424
+ saved = await r.eval(
425
+ FENCE_SCRIPT, 2,
426
+ f"fence:{task_id}", queueing.task_key(task_id),
427
+ str(my_token), task_id, queueing.SUCCESS, queueing.encode_result(result),
428
+ )
429
+
430
+ if saved:
431
+ finished_at = time.time()
432
+ async with r.pipeline(transaction=True) as pipe:
433
+ pipe.hset(queueing.task_key(task_id), mapping={
434
+ "finished_at": finished_at,
435
+ "duration_ms": round((finished_at - started_at) * 1000, 1),
436
+ "error": "",
437
+ })
438
+ pipe.expire(queueing.task_key(task_id), settings.RESULT_TTL)
439
+ pipe.incr("stats:completed")
440
+ pipe.hincrby(f"worker:{WORKER_ID}", "processed", 1)
441
+ await pipe.execute()
442
+ span.set_attribute("task.status", "Success")
443
+ logger.info(f"Task {task_id} completed successfully.")
444
+ # ── Webhook delivery ──────────────────────────────────────
445
+ # Travels through the queue like any other task, so network
446
+ # blips are retried automatically.
447
+ if tasks.on_success_url:
448
+ webhook = queueing.new_payload(
449
+ "_deliver_webhook",
450
+ [tasks.on_success_url, task_id, tasks.task_name, queueing.encode_result(result)],
451
+ )
452
+ await queueing.enqueue(r, webhook)
453
+ logger.info(f"Enqueued webhook delivery for task {task_id} → {tasks.on_success_url}")
454
+ else:
455
+ span.set_attribute("task.status", "Discarded")
456
+ span.set_attribute("task.discard_reason", "fence_token_stale")
457
+ logger.warning(f"Task {task_id} discarded (stale fence token).")
458
+
459
+ except Exception as e:
460
+ error = f"{type(e).__name__}: {e}"
461
+ logger.error(f"Task {task_id} failed: {error}")
462
+ span.set_status(trace.StatusCode.ERROR, str(e))
463
+ span.record_exception(e)
464
+ finished_at = time.time()
465
+ await r.incr("stats:failed")
466
+ await r.hincrby(f"worker:{WORKER_ID}", "failed", 1)
467
+
468
+ if tasks.retry_count >= max_retries:
469
+ logger.error(f"[DLQ] Task {task_id} failed after {tasks.retry_count} retries. Moving to DLQ.")
470
+ span.set_attribute("task.status", "DeadLetter")
471
+ dlq_entry = json.loads(task_json)
472
+ dlq_entry["fence_token"] = tasks.fence_token
473
+ dlq_entry["error"] = error
474
+ dlq_entry["failed_at"] = finished_at
475
+ async with r.pipeline(transaction=True) as pipe:
476
+ pipe.lpush("dlq", json.dumps(dlq_entry))
477
+ pipe.incr("stats:dlq")
478
+ pipe.hset(queueing.task_key(task_id), mapping={
479
+ "task_id": task_id,
480
+ "status": queueing.DEAD_LETTER,
481
+ "finished_at": finished_at,
482
+ "error": f"Failed after {tasks.retry_count} retries. Last error: {error}",
483
+ })
484
+ await pipe.execute()
485
+ else:
486
+ tasks.retry_count += 1
487
+ BASE = 1.0
488
+ MAX_DELAY = 30.0
489
+ prev = getattr(tasks, "prev_delay", BASE)
490
+ delay = min(MAX_DELAY, random.uniform(BASE, prev * 3))
491
+ tasks.prev_delay = delay
492
+ retry_raw = json.dumps(tasks.model_dump())
493
+
494
+ async with r.pipeline(transaction=True) as pipe:
495
+ pipe.zadd("delayed_tasks", {retry_raw: time.time() + delay})
496
+ pipe.incr("stats:delayed")
497
+ pipe.hset(queueing.task_key(task_id), mapping={
498
+ "task_id": task_id,
499
+ "status": queueing.RETRY_SCHEDULED,
500
+ "retry_count": tasks.retry_count,
501
+ "next_retry_at": time.time() + delay,
502
+ "payload": retry_raw,
503
+ "error": f"{error}. Retry {tasks.retry_count}/{max_retries} in {delay:.1f}s.",
504
+ })
505
+ await pipe.execute()
506
+
507
+ finally:
508
+ try:
509
+ async with r.pipeline(transaction=True) as pipe:
510
+ pipe.lrem(f"processing_queue:{WORKER_ID}", count=1, value=task_json)
511
+ if incr_done:
512
+ pipe.decr("stats:processing")
513
+ pipe.publish(queueing.EVENTS_CHANNEL, "update")
514
+ await pipe.execute()
515
+ except Exception as e:
516
+ logger.error(f"[Cleanup] Redis error: {e}")
517
+ finally:
518
+ sem.release()
519
+
520
+
521
+ async def _recover_own_tasks():
522
+ """On boot, put back anything this worker id left in its processing queue."""
523
+ while True:
524
+ leftover = await redis_client.r.rpoplpush(f"processing_queue:{WORKER_ID}", "queue:high")
525
+ if not leftover:
526
+ break
527
+ await redis_client.r.incr("stats:pending")
528
+ await redis_client.r.decr("stats:processing")
529
+ logger.info(f"Recovered crashed task on startup: {leftover}")
530
+
531
+
532
+ async def consumer_task():
533
+ concurrency = CPU_WORKERS + IO_WORKERS
534
+ logger.info(
535
+ f"Worker {WORKER_ID}: {_TOTAL_CORES} CPU cores detected. "
536
+ f"CPU processes={CPU_WORKERS}, IO threads={IO_WORKERS}, max concurrent tasks={concurrency}"
537
+ )
538
+ user_tasks = sorted(t for t in TASKS if not t.startswith("_"))
539
+ logger.info(f"Registered tasks ({len(user_tasks)}): {', '.join(user_tasks) or 'none'}")
540
+
541
+ process_pool = ProcessPoolExecutor(max_workers=CPU_WORKERS)
542
+ thread_pool = ThreadPoolExecutor(max_workers=IO_WORKERS)
543
+ loop = asyncio.get_running_loop()
544
+ sem = asyncio.Semaphore(concurrency)
545
+
546
+ def _signal_handler():
547
+ logger.info("Shutdown signal received. Finishing running tasks...")
548
+ shutdown_event.set()
549
+
550
+ try:
551
+ loop.add_signal_handler(signal.SIGINT, _signal_handler)
552
+ loop.add_signal_handler(signal.SIGTERM, _signal_handler)
553
+ except NotImplementedError: # Windows
554
+ signal.signal(signal.SIGINT, lambda s, f: loop.call_soon_threadsafe(_signal_handler))
555
+ signal.signal(signal.SIGTERM, lambda s, f: loop.call_soon_threadsafe(_signal_handler))
556
+
557
+ while not shutdown_event.is_set():
558
+ try:
559
+ await _recover_own_tasks()
560
+ await redis_client.r.hset(f"worker:{WORKER_ID}", mapping={**_worker_info(), "started_at": time.time()})
561
+ await _publish_registry()
562
+ break
563
+ except Exception as e:
564
+ logger.error(f"[Startup] Redis unavailable ({e.__class__.__name__}). Retrying in 5s...")
565
+ await asyncio.sleep(5)
566
+
567
+ background = [
568
+ asyncio.create_task(cron_scheduler()),
569
+ asyncio.create_task(retry_scheduler()),
570
+ asyncio.create_task(heartbeat()),
571
+ asyncio.create_task(zombie_sweeper()),
572
+ ]
573
+
574
+ logger.info(f"Worker {WORKER_ID} started and ready to process tasks.")
575
+
576
+ idle_delay = 0.05
577
+ max_idle_delay = 1.0
578
+
579
+ while not shutdown_event.is_set():
580
+ try:
581
+ await sem.acquire()
582
+ task_json = await redis_client.r.eval(
583
+ PRIORITY_POP_SCRIPT,
584
+ 4,
585
+ f"processing_queue:{WORKER_ID}",
586
+ "queue:high",
587
+ "queue:default",
588
+ "queue:low",
589
+ )
590
+
591
+ if not task_json:
592
+ sem.release()
593
+ try:
594
+ await asyncio.wait_for(shutdown_event.wait(), timeout=idle_delay)
595
+ except asyncio.TimeoutError:
596
+ pass
597
+ idle_delay = min(idle_delay * 1.5, max_idle_delay)
598
+ continue
599
+
600
+ idle_delay = 0.05
601
+ except Exception as e:
602
+ sem.release()
603
+ logger.error(f"[Main loop] Redis error: {e.__class__.__name__}. Backing off 5s...")
604
+ await asyncio.sleep(5)
605
+ continue
606
+
607
+ task = asyncio.create_task(
608
+ handle_task(task_json, sem, loop, process_pool, thread_pool)
609
+ )
610
+ active_tasks.add(task)
611
+ task.add_done_callback(active_tasks.discard)
612
+
613
+ if active_tasks:
614
+ logger.info(f"Waiting for {len(active_tasks)} active tasks to complete...")
615
+ await asyncio.gather(*active_tasks, return_exceptions=True)
616
+
617
+ for bg in background:
618
+ bg.cancel()
619
+ try:
620
+ await redis_client.r.zrem("active_workers", WORKER_ID)
621
+ await redis_client.r.delete(f"worker:{WORKER_ID}")
622
+ except Exception:
623
+ pass
624
+
625
+ thread_pool.shutdown(wait=True)
626
+ process_pool.shutdown(wait=True)
627
+ await redis_client.r.aclose()
628
+
629
+ provider = trace.get_tracer_provider()
630
+ if hasattr(provider, "force_flush"):
631
+ provider.force_flush()
632
+ logger.info(f"Worker {WORKER_ID} stopped cleanly.")
633
+
634
+
635
+ if __name__ == "__main__":
636
+ asyncio.run(consumer_task())