broccoli-workers 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- broccoli/__init__.py +0 -0
- broccoli/cli.py +637 -0
- broccoli/core/__init__.py +15 -0
- broccoli/core/chain/chain.py +45 -0
- broccoli/core/chain/chain_mixin.py +49 -0
- broccoli/core/chain/chain_queue.py +299 -0
- broccoli/core/chain/task_chain.py +201 -0
- broccoli/core/health.py +20 -0
- broccoli/core/redis_controller.py +35 -0
- broccoli/core/result.py +78 -0
- broccoli/core/task/task.py +58 -0
- broccoli/core/task/task_queue.py +300 -0
- broccoli/core/task/task_registry.py +47 -0
- broccoli/workers/__init__.py +15 -0
- broccoli/workers/async_worker.py +157 -0
- broccoli/workers/base_worker.py +339 -0
- broccoli/workers/chain_worker.py +94 -0
- broccoli/workers/hybrid_worker.py +269 -0
- broccoli/workers/threaded_worker.py +154 -0
- broccoli/workers/worker_pool.py +99 -0
- broccoli_workers-0.1.0.dist-info/METADATA +594 -0
- broccoli_workers-0.1.0.dist-info/RECORD +26 -0
- broccoli_workers-0.1.0.dist-info/WHEEL +5 -0
- broccoli_workers-0.1.0.dist-info/entry_points.txt +2 -0
- broccoli_workers-0.1.0.dist-info/licenses/LICENSE +21 -0
- broccoli_workers-0.1.0.dist-info/top_level.txt +1 -0
broccoli/__init__.py
ADDED
|
File without changes
|
broccoli/cli.py
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Broccoli Task Queue CLI
|
|
4
|
+
|
|
5
|
+
Usage examples:
|
|
6
|
+
# Start a threaded worker pool (4 workers)
|
|
7
|
+
broccoli worker start --type threaded --pool --num-workers 4
|
|
8
|
+
|
|
9
|
+
# Start a single async worker with 10 concurrent tasks
|
|
10
|
+
broccoli worker start --type async --concurrency 10
|
|
11
|
+
|
|
12
|
+
# Start a chain worker (single)
|
|
13
|
+
broccoli worker start --type chain
|
|
14
|
+
|
|
15
|
+
# Inspect queue stats
|
|
16
|
+
broccoli queue stats
|
|
17
|
+
|
|
18
|
+
# List pending tasks
|
|
19
|
+
broccoli queue list --status pending --limit 5
|
|
20
|
+
|
|
21
|
+
# Get a task by ID
|
|
22
|
+
broccoli queue get <task_id>
|
|
23
|
+
|
|
24
|
+
# See which tasks are waiting for a given parent
|
|
25
|
+
broccoli queue waiting <parent_id>
|
|
26
|
+
|
|
27
|
+
# List dead-letter tasks
|
|
28
|
+
broccoli dead list
|
|
29
|
+
|
|
30
|
+
# Requeue a dead task (retry it)
|
|
31
|
+
broccoli dead requeue <task_id>
|
|
32
|
+
|
|
33
|
+
# Get chain status
|
|
34
|
+
broccoli chain status <chain_id>
|
|
35
|
+
|
|
36
|
+
# List tasks in a chain
|
|
37
|
+
broccoli chain tasks <chain_id>
|
|
38
|
+
|
|
39
|
+
# Health check (returns exit code 0 if Redis is reachable)
|
|
40
|
+
broccoli health
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
import argparse
|
|
44
|
+
import json
|
|
45
|
+
import logging
|
|
46
|
+
import os
|
|
47
|
+
import sys
|
|
48
|
+
from typing import Any, Dict, List, Optional
|
|
49
|
+
|
|
50
|
+
from broccoli.core.chain.chain_queue import ChainQueue
|
|
51
|
+
from broccoli.core.chain.task_chain import TaskChain
|
|
52
|
+
|
|
53
|
+
# Import Broccoli components
|
|
54
|
+
from broccoli.core.redis_controller import RedisController
|
|
55
|
+
from broccoli.core.task.task import Task
|
|
56
|
+
from broccoli.core.task.task_queue import TaskQueue
|
|
57
|
+
from broccoli.workers.async_worker import AsyncWorker
|
|
58
|
+
from broccoli.workers.base_worker import BaseWorker
|
|
59
|
+
from broccoli.workers.chain_worker import ChainWorker
|
|
60
|
+
from broccoli.workers.hybrid_worker import HybridWorker
|
|
61
|
+
from broccoli.workers.threaded_worker import ThreadedWorker
|
|
62
|
+
from broccoli.workers.worker_pool import WorkerPool
|
|
63
|
+
|
|
64
|
+
# Defaults from environment variables
|
|
65
|
+
DEFAULT_REDIS_URL = os.getenv("BROCCOLI_REDIS_URL", "redis://localhost:6379")
|
|
66
|
+
DEFAULT_QUEUE_NAME = os.getenv("BROCCOLI_QUEUE_NAME", "tasks:queue")
|
|
67
|
+
DEFAULT_CHAIN_QUEUE_NAME = os.getenv("BROCCOLI_CHAIN_QUEUE_NAME", "chain_tasks:queue")
|
|
68
|
+
DEFAULT_TASK_PREFIX = os.getenv("BROCCOLI_TASK_PREFIX", "task")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def setup_logging(verbose: int):
|
|
72
|
+
"""Configure logging based on verbosity count."""
|
|
73
|
+
level = logging.WARNING
|
|
74
|
+
if verbose >= 2:
|
|
75
|
+
level = logging.DEBUG
|
|
76
|
+
elif verbose == 1:
|
|
77
|
+
level = logging.INFO
|
|
78
|
+
logging.basicConfig(
|
|
79
|
+
level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def print_table(headers: List[str], rows: List[List[Any]]):
|
|
84
|
+
"""Pretty-print a table with aligned columns."""
|
|
85
|
+
if not rows:
|
|
86
|
+
print("No data.")
|
|
87
|
+
return
|
|
88
|
+
col_widths = [len(h) for h in headers]
|
|
89
|
+
for row in rows:
|
|
90
|
+
for i, cell in enumerate(row):
|
|
91
|
+
col_widths[i] = max(col_widths[i], len(str(cell)))
|
|
92
|
+
fmt = " ".join(f"{{:<{w}}}" for w in col_widths)
|
|
93
|
+
print(fmt.format(*headers))
|
|
94
|
+
print("-" * (sum(col_widths) + 2 * (len(headers) - 1)))
|
|
95
|
+
for row in rows:
|
|
96
|
+
print(fmt.format(*(str(c) for c in row)))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def output_json(data: Any):
|
|
100
|
+
"""Print JSON with indentation."""
|
|
101
|
+
print(json.dumps(data, indent=2, default=str))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def output_result(data: Any, fmt: str):
|
|
105
|
+
"""Print data in either JSON or table format."""
|
|
106
|
+
if fmt == "json":
|
|
107
|
+
output_json(data)
|
|
108
|
+
else:
|
|
109
|
+
if isinstance(data, list) and data and isinstance(data[0], dict):
|
|
110
|
+
# Try to render as table using dict keys
|
|
111
|
+
headers = list(data[0].keys())
|
|
112
|
+
rows = [[row.get(k, "") for k in headers] for row in data]
|
|
113
|
+
print_table(headers, rows)
|
|
114
|
+
elif isinstance(data, dict):
|
|
115
|
+
# Print key-value pairs
|
|
116
|
+
for k, v in data.items():
|
|
117
|
+
print(f"{k}: {v}")
|
|
118
|
+
else:
|
|
119
|
+
print(data)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def get_queue(args) -> TaskQueue:
|
|
123
|
+
"""Instantiate a TaskQueue with CLI args."""
|
|
124
|
+
return TaskQueue(
|
|
125
|
+
redis_url=args.redis_url,
|
|
126
|
+
queue_name=args.queue_name,
|
|
127
|
+
task_prefix=args.task_prefix,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def get_chain_queue(args) -> ChainQueue:
|
|
132
|
+
"""Instantiate a ChainQueue with CLI args."""
|
|
133
|
+
return ChainQueue(
|
|
134
|
+
redis_url=args.redis_url,
|
|
135
|
+
queue_name=args.chain_queue_name,
|
|
136
|
+
task_prefix=args.task_prefix,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def get_task_chain(args) -> TaskChain:
|
|
141
|
+
"""Instantiate a TaskChain with CLI args."""
|
|
142
|
+
return TaskChain(redis_url=args.redis_url)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ============ Worker start ============
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def cmd_worker_start(args):
|
|
149
|
+
"""Start a worker or worker pool."""
|
|
150
|
+
setup_logging(args.verbose)
|
|
151
|
+
|
|
152
|
+
# Map worker types to classes
|
|
153
|
+
worker_classes = {
|
|
154
|
+
"base": BaseWorker,
|
|
155
|
+
"threaded": ThreadedWorker,
|
|
156
|
+
"async": AsyncWorker,
|
|
157
|
+
"hybrid": HybridWorker,
|
|
158
|
+
"chain": ChainWorker,
|
|
159
|
+
}
|
|
160
|
+
worker_class = worker_classes[args.type]
|
|
161
|
+
|
|
162
|
+
# Determine which queue name and task prefix to use
|
|
163
|
+
if args.type == "chain":
|
|
164
|
+
queue_name = args.chain_queue_name
|
|
165
|
+
else:
|
|
166
|
+
queue_name = args.queue_name
|
|
167
|
+
|
|
168
|
+
# Common worker kwargs
|
|
169
|
+
worker_kwargs = {
|
|
170
|
+
"redis_url": args.redis_url,
|
|
171
|
+
"worker_id": args.worker_id,
|
|
172
|
+
"queue_name": queue_name,
|
|
173
|
+
"task_prefix": args.task_prefix,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
# Add type-specific kwargs
|
|
177
|
+
if args.type == "threaded":
|
|
178
|
+
worker_kwargs["max_workers"] = args.concurrency
|
|
179
|
+
elif args.type == "async":
|
|
180
|
+
worker_kwargs["max_concurrent"] = args.concurrency
|
|
181
|
+
elif args.type == "hybrid":
|
|
182
|
+
worker_kwargs["thread_workers"] = args.thread_workers
|
|
183
|
+
worker_kwargs["async_tasks"] = args.async_tasks
|
|
184
|
+
# ChainWorker inherits BaseWorker and uses default args; no extra needed
|
|
185
|
+
|
|
186
|
+
# Optionally recover stalled tasks before starting
|
|
187
|
+
if args.recover_stalled > 0:
|
|
188
|
+
# We need a queue instance to call recover_stalled
|
|
189
|
+
temp_queue = TaskQueue(
|
|
190
|
+
redis_url=args.redis_url,
|
|
191
|
+
queue_name=queue_name,
|
|
192
|
+
task_prefix=args.task_prefix,
|
|
193
|
+
)
|
|
194
|
+
recovered = temp_queue.recover_stalled(args.recover_stalled)
|
|
195
|
+
logging.info(
|
|
196
|
+
f"Recovered {recovered} stalled tasks (timeout={args.recover_stalled}s)"
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
if args.pool:
|
|
200
|
+
pool = WorkerPool(
|
|
201
|
+
worker_type=worker_class,
|
|
202
|
+
num_workers=args.num_workers,
|
|
203
|
+
redis_url=args.redis_url,
|
|
204
|
+
**worker_kwargs,
|
|
205
|
+
)
|
|
206
|
+
pool.start()
|
|
207
|
+
else:
|
|
208
|
+
worker = worker_class(**worker_kwargs)
|
|
209
|
+
worker.start()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ============ Queue commands ============
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def cmd_queue_stats(args):
|
|
216
|
+
"""Show queue statistics."""
|
|
217
|
+
q = get_queue(args)
|
|
218
|
+
stats = q.stats()
|
|
219
|
+
# Also get waiting tasks count? Not directly available; we can count all dependency keys?
|
|
220
|
+
# We'll just show what we have.
|
|
221
|
+
output_result(stats, args.format)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def cmd_queue_list(args):
|
|
225
|
+
"""List task IDs matching a status."""
|
|
226
|
+
q = get_queue(args)
|
|
227
|
+
# We need to scan all task hashes or use a status index? Not available.
|
|
228
|
+
# We can use Redis SCAN with pattern to find tasks by status? That's expensive.
|
|
229
|
+
# Instead, we can use the queue sorted set for pending tasks, processing set for in_progress,
|
|
230
|
+
# and we can scan for waiting status? There's no index for waiting.
|
|
231
|
+
# We'll provide a limited set: pending (from queue), in_progress (from processing), and completed/failed/waiting are not directly listable without scanning all task hashes.
|
|
232
|
+
# For simplicity, we'll only support pending and in_progress, and maybe we can scan all task keys for a demo.
|
|
233
|
+
# Better: we'll implement a scan over `task:*` and filter by status. This is not efficient but works for small/medium workloads.
|
|
234
|
+
if args.status not in (
|
|
235
|
+
"pending",
|
|
236
|
+
"in_progress",
|
|
237
|
+
"completed",
|
|
238
|
+
"failed",
|
|
239
|
+
"waiting",
|
|
240
|
+
"all",
|
|
241
|
+
):
|
|
242
|
+
print(
|
|
243
|
+
f"Invalid status: {args.status}. Must be one of: pending, in_progress, completed, failed, waiting, all",
|
|
244
|
+
file=sys.stderr,
|
|
245
|
+
)
|
|
246
|
+
sys.exit(1)
|
|
247
|
+
|
|
248
|
+
redis_client = RedisController(args.redis_url).get_client()
|
|
249
|
+
task_keys = redis_client.keys(f"{args.task_prefix}:*")
|
|
250
|
+
tasks = []
|
|
251
|
+
for key in task_keys:
|
|
252
|
+
# key is bytes or str
|
|
253
|
+
task_data = redis_client.hgetall(key)
|
|
254
|
+
if not task_data:
|
|
255
|
+
continue
|
|
256
|
+
# decode bytes if needed
|
|
257
|
+
if isinstance(task_data, dict):
|
|
258
|
+
# Convert bytes keys/values to str
|
|
259
|
+
decoded = {}
|
|
260
|
+
for k, v in task_data.items():
|
|
261
|
+
if isinstance(k, bytes):
|
|
262
|
+
k = k.decode()
|
|
263
|
+
if isinstance(v, bytes):
|
|
264
|
+
v = v.decode()
|
|
265
|
+
decoded[k] = v
|
|
266
|
+
task_data = decoded
|
|
267
|
+
status = task_data.get("status")
|
|
268
|
+
if args.status != "all" and status != args.status:
|
|
269
|
+
continue
|
|
270
|
+
tasks.append(
|
|
271
|
+
{
|
|
272
|
+
"task_id": task_data.get("task_id"),
|
|
273
|
+
"task_type": task_data.get("task_type"),
|
|
274
|
+
"status": status,
|
|
275
|
+
"created_at": task_data.get("created_at"),
|
|
276
|
+
}
|
|
277
|
+
)
|
|
278
|
+
# Sort by created_at
|
|
279
|
+
tasks.sort(key=lambda x: x.get("created_at", ""))
|
|
280
|
+
if args.limit and len(tasks) > args.limit:
|
|
281
|
+
tasks = tasks[: args.limit]
|
|
282
|
+
output_result(tasks, args.format)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def cmd_queue_get(args):
|
|
286
|
+
"""Fetch and display a task by ID."""
|
|
287
|
+
q = get_queue(args)
|
|
288
|
+
task = q.get_task(args.task_id)
|
|
289
|
+
if not task:
|
|
290
|
+
print(f"Task {args.task_id} not found", file=sys.stderr)
|
|
291
|
+
sys.exit(1)
|
|
292
|
+
# Convert to dict for output
|
|
293
|
+
data = task.to_dict()
|
|
294
|
+
output_result(data, args.format)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def cmd_queue_waiting(args):
|
|
298
|
+
"""Show tasks waiting for a specific parent."""
|
|
299
|
+
q = get_queue(args)
|
|
300
|
+
waiting_ids = q.get_waiting_for(args.parent_id)
|
|
301
|
+
if not waiting_ids:
|
|
302
|
+
print(f"No tasks waiting for {args.parent_id}")
|
|
303
|
+
return
|
|
304
|
+
# Optionally fetch each task's details
|
|
305
|
+
tasks = []
|
|
306
|
+
for wid in waiting_ids:
|
|
307
|
+
task = q.get_task(wid)
|
|
308
|
+
if task:
|
|
309
|
+
tasks.append(
|
|
310
|
+
{
|
|
311
|
+
"task_id": task.task_id,
|
|
312
|
+
"task_type": task.task_type,
|
|
313
|
+
"status": task.status,
|
|
314
|
+
"created_at": task.created_at,
|
|
315
|
+
}
|
|
316
|
+
)
|
|
317
|
+
output_result(tasks, args.format)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
# ============ Dead-letter commands ============
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def cmd_dead_list(args):
|
|
324
|
+
"""List dead-letter tasks."""
|
|
325
|
+
redis_client = RedisController(args.redis_url).get_client()
|
|
326
|
+
dead_key = f"{args.task_prefix}:dead_letter"
|
|
327
|
+
# zrange to get all with scores (timestamps)
|
|
328
|
+
members = redis_client.zrange(dead_key, 0, -1, withscores=True)
|
|
329
|
+
if not members:
|
|
330
|
+
print("No dead-letter tasks.")
|
|
331
|
+
return
|
|
332
|
+
tasks = []
|
|
333
|
+
for member, score in members:
|
|
334
|
+
task_id = member.decode() if isinstance(member, bytes) else member
|
|
335
|
+
# Try to fetch the original task data (may have been cleaned up, but we can store copy in dead letter)
|
|
336
|
+
# We'll just show the ID and timestamp.
|
|
337
|
+
tasks.append(
|
|
338
|
+
{
|
|
339
|
+
"task_id": task_id,
|
|
340
|
+
"failed_at": score, # timestamp
|
|
341
|
+
}
|
|
342
|
+
)
|
|
343
|
+
output_result(tasks, args.format)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def cmd_dead_requeue(args):
|
|
347
|
+
"""Requeue a dead-letter task (retry it)."""
|
|
348
|
+
redis_client = RedisController(args.redis_url).get_client()
|
|
349
|
+
dead_key = f"{args.task_prefix}:dead_letter"
|
|
350
|
+
# Check if task is in dead set
|
|
351
|
+
is_dead = redis_client.zscore(dead_key, args.task_id) is not None
|
|
352
|
+
if not is_dead:
|
|
353
|
+
print(f"Task {args.task_id} not found in dead-letter set", file=sys.stderr)
|
|
354
|
+
sys.exit(1)
|
|
355
|
+
# We need to re-push it to the queue. But we need the task data. It may have been deleted.
|
|
356
|
+
# We can retrieve from result backend? Or we can store a copy.
|
|
357
|
+
# For simplicity, we'll assume the task hash still exists (if not, we can't).
|
|
358
|
+
q = get_queue(args)
|
|
359
|
+
task = q.get_task(args.task_id)
|
|
360
|
+
if not task:
|
|
361
|
+
print(
|
|
362
|
+
f"Task {args.task_id} metadata not found; cannot requeue. (Maybe it was already cleaned up.)",
|
|
363
|
+
file=sys.stderr,
|
|
364
|
+
)
|
|
365
|
+
sys.exit(1)
|
|
366
|
+
# Reset retries to 0 and push again
|
|
367
|
+
task.retries = 0
|
|
368
|
+
task.status = "pending"
|
|
369
|
+
task.error = None
|
|
370
|
+
# Remove from dead-letter set
|
|
371
|
+
redis_client.zrem(dead_key, args.task_id)
|
|
372
|
+
# Push back to queue
|
|
373
|
+
q.push(task)
|
|
374
|
+
print(f"Requeued task {args.task_id}")
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# ============ Chain commands ============
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def cmd_chain_status(args):
|
|
381
|
+
"""Get chain status."""
|
|
382
|
+
tc = get_task_chain(args)
|
|
383
|
+
status = tc.get_chain_status(args.chain_id)
|
|
384
|
+
output_result(status, args.format)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def cmd_chain_tasks(args):
|
|
388
|
+
"""List tasks in a chain."""
|
|
389
|
+
redis_client = RedisController(args.redis_url).get_client()
|
|
390
|
+
tasks_json = redis_client.get(f"{args.task_prefix}:{args.chain_id}:tasks")
|
|
391
|
+
if not tasks_json:
|
|
392
|
+
print(f"No tasks found for chain {args.chain_id}", file=sys.stderr)
|
|
393
|
+
sys.exit(1)
|
|
394
|
+
tasks = json.loads(tasks_json)
|
|
395
|
+
output_result(tasks, args.format)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
# ============ Health check ============
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def cmd_health(args):
|
|
402
|
+
"""Check if Redis is reachable and basic queue operations work."""
|
|
403
|
+
try:
|
|
404
|
+
redis_client = RedisController(args.redis_url).get_client()
|
|
405
|
+
redis_client.ping()
|
|
406
|
+
# Also try to get queue stats
|
|
407
|
+
q = get_queue(args)
|
|
408
|
+
q.stats()
|
|
409
|
+
print("OK")
|
|
410
|
+
sys.exit(0)
|
|
411
|
+
except Exception as e:
|
|
412
|
+
print(f"ERROR: {e}", file=sys.stderr)
|
|
413
|
+
sys.exit(1)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
# ============ Main parser ============
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def create_parser():
|
|
420
|
+
parser = argparse.ArgumentParser(
|
|
421
|
+
description="Broccoli Task Queue CLI",
|
|
422
|
+
epilog="See subcommand help for details: broccoli <command> --help",
|
|
423
|
+
)
|
|
424
|
+
parser.add_argument(
|
|
425
|
+
"--verbose",
|
|
426
|
+
"-v",
|
|
427
|
+
action="count",
|
|
428
|
+
default=0,
|
|
429
|
+
help="Increase logging verbosity (-v for INFO, -vv for DEBUG)",
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
subparsers = parser.add_subparsers(
|
|
433
|
+
dest="command", required=True, help="Subcommand to run"
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
# ---------- worker start ----------
|
|
437
|
+
worker_parser = subparsers.add_parser("worker", help="Manage workers")
|
|
438
|
+
worker_subparsers = worker_parser.add_subparsers(
|
|
439
|
+
dest="worker_action", required=True
|
|
440
|
+
)
|
|
441
|
+
start_parser = worker_subparsers.add_parser("start", help="Start a worker or pool")
|
|
442
|
+
start_parser.add_argument(
|
|
443
|
+
"--type",
|
|
444
|
+
choices=["base", "threaded", "async", "hybrid", "chain"],
|
|
445
|
+
default="threaded",
|
|
446
|
+
help="Worker type (default: threaded)",
|
|
447
|
+
)
|
|
448
|
+
start_parser.add_argument(
|
|
449
|
+
"--pool", action="store_true", help="Run multiple workers in a pool"
|
|
450
|
+
)
|
|
451
|
+
start_parser.add_argument(
|
|
452
|
+
"--num-workers",
|
|
453
|
+
type=int,
|
|
454
|
+
default=4,
|
|
455
|
+
help="Number of workers in pool (default: 4)",
|
|
456
|
+
)
|
|
457
|
+
start_parser.add_argument(
|
|
458
|
+
"--redis-url",
|
|
459
|
+
default=DEFAULT_REDIS_URL,
|
|
460
|
+
help=f"Redis URL (env: BROCCOLI_REDIS_URL, default: {DEFAULT_REDIS_URL})",
|
|
461
|
+
)
|
|
462
|
+
start_parser.add_argument(
|
|
463
|
+
"--queue-name",
|
|
464
|
+
default=DEFAULT_QUEUE_NAME,
|
|
465
|
+
help=f"Queue name for regular tasks (env: BROCCOLI_QUEUE_NAME, default: {DEFAULT_QUEUE_NAME})",
|
|
466
|
+
)
|
|
467
|
+
start_parser.add_argument(
|
|
468
|
+
"--chain-queue-name",
|
|
469
|
+
default=DEFAULT_CHAIN_QUEUE_NAME,
|
|
470
|
+
help=f"Queue name for chain tasks (env: BROCCOLI_CHAIN_QUEUE_NAME, default: {DEFAULT_CHAIN_QUEUE_NAME})",
|
|
471
|
+
)
|
|
472
|
+
start_parser.add_argument(
|
|
473
|
+
"--task-prefix",
|
|
474
|
+
default=DEFAULT_TASK_PREFIX,
|
|
475
|
+
help=f"Task hash prefix (env: BROCCOLI_TASK_PREFIX, default: {DEFAULT_TASK_PREFIX})",
|
|
476
|
+
)
|
|
477
|
+
start_parser.add_argument(
|
|
478
|
+
"--worker-id", help="Unique worker ID (auto-generated if not set)"
|
|
479
|
+
)
|
|
480
|
+
start_parser.add_argument(
|
|
481
|
+
"--concurrency",
|
|
482
|
+
type=int,
|
|
483
|
+
default=4,
|
|
484
|
+
help="Concurrency level (threads for threaded/hybrid, async tasks for async)",
|
|
485
|
+
)
|
|
486
|
+
start_parser.add_argument(
|
|
487
|
+
"--thread-workers",
|
|
488
|
+
type=int,
|
|
489
|
+
default=4,
|
|
490
|
+
help="Thread pool size for hybrid worker (default: 4)",
|
|
491
|
+
)
|
|
492
|
+
start_parser.add_argument(
|
|
493
|
+
"--async-tasks",
|
|
494
|
+
type=int,
|
|
495
|
+
default=10,
|
|
496
|
+
help="Async task concurrency for hybrid worker (default: 10)",
|
|
497
|
+
)
|
|
498
|
+
start_parser.add_argument(
|
|
499
|
+
"--recover-stalled",
|
|
500
|
+
type=int,
|
|
501
|
+
default=0,
|
|
502
|
+
help="Recover stalled tasks older than N seconds before starting (0 = off)",
|
|
503
|
+
)
|
|
504
|
+
start_parser.set_defaults(func=cmd_worker_start)
|
|
505
|
+
|
|
506
|
+
# ---------- queue ----------
|
|
507
|
+
queue_parser = subparsers.add_parser("queue", help="Inspect the task queue")
|
|
508
|
+
queue_subparsers = queue_parser.add_subparsers(dest="queue_action", required=True)
|
|
509
|
+
|
|
510
|
+
# queue stats
|
|
511
|
+
stats_parser = queue_subparsers.add_parser("stats", help="Show queue statistics")
|
|
512
|
+
stats_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
513
|
+
stats_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
514
|
+
stats_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
515
|
+
stats_parser.add_argument(
|
|
516
|
+
"--format", choices=["table", "json"], default="table", help="Output format"
|
|
517
|
+
)
|
|
518
|
+
stats_parser.set_defaults(func=cmd_queue_stats)
|
|
519
|
+
|
|
520
|
+
# queue list
|
|
521
|
+
list_parser = queue_subparsers.add_parser("list", help="List tasks by status")
|
|
522
|
+
list_parser.add_argument(
|
|
523
|
+
"--status",
|
|
524
|
+
default="pending",
|
|
525
|
+
choices=["pending", "in_progress", "completed", "failed", "waiting", "all"],
|
|
526
|
+
help="Filter by status",
|
|
527
|
+
)
|
|
528
|
+
list_parser.add_argument(
|
|
529
|
+
"--limit", type=int, help="Maximum number of tasks to show"
|
|
530
|
+
)
|
|
531
|
+
list_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
532
|
+
list_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
533
|
+
list_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
534
|
+
list_parser.add_argument("--format", choices=["table", "json"], default="table")
|
|
535
|
+
list_parser.set_defaults(func=cmd_queue_list)
|
|
536
|
+
|
|
537
|
+
# queue get
|
|
538
|
+
get_parser = queue_subparsers.add_parser("get", help="Get task details by ID")
|
|
539
|
+
get_parser.add_argument("task_id", help="Task ID")
|
|
540
|
+
get_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
541
|
+
get_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
542
|
+
get_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
543
|
+
get_parser.add_argument("--format", choices=["table", "json"], default="table")
|
|
544
|
+
get_parser.set_defaults(func=cmd_queue_get)
|
|
545
|
+
|
|
546
|
+
# queue waiting
|
|
547
|
+
waiting_parser = queue_subparsers.add_parser(
|
|
548
|
+
"waiting", help="Show tasks waiting for a parent task"
|
|
549
|
+
)
|
|
550
|
+
waiting_parser.add_argument("parent_id", help="Parent task ID")
|
|
551
|
+
waiting_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
552
|
+
waiting_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
553
|
+
waiting_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
554
|
+
waiting_parser.add_argument("--format", choices=["table", "json"], default="table")
|
|
555
|
+
waiting_parser.set_defaults(func=cmd_queue_waiting)
|
|
556
|
+
|
|
557
|
+
# ---------- dead ----------
|
|
558
|
+
dead_parser = subparsers.add_parser("dead", help="Manage dead-letter tasks")
|
|
559
|
+
dead_subparsers = dead_parser.add_subparsers(dest="dead_action", required=True)
|
|
560
|
+
|
|
561
|
+
dead_list_parser = dead_subparsers.add_parser("list", help="List dead-letter tasks")
|
|
562
|
+
dead_list_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
563
|
+
dead_list_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
564
|
+
dead_list_parser.add_argument(
|
|
565
|
+
"--format", choices=["table", "json"], default="table"
|
|
566
|
+
)
|
|
567
|
+
dead_list_parser.set_defaults(func=cmd_dead_list)
|
|
568
|
+
|
|
569
|
+
dead_requeue_parser = dead_subparsers.add_parser(
|
|
570
|
+
"requeue", help="Requeue a dead-letter task (retry)"
|
|
571
|
+
)
|
|
572
|
+
dead_requeue_parser.add_argument("task_id", help="Task ID to requeue")
|
|
573
|
+
dead_requeue_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
574
|
+
dead_requeue_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
575
|
+
dead_requeue_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
576
|
+
dead_requeue_parser.set_defaults(func=cmd_dead_requeue)
|
|
577
|
+
|
|
578
|
+
# ---------- chain ----------
|
|
579
|
+
chain_parser = subparsers.add_parser("chain", help="Inspect task chains")
|
|
580
|
+
chain_subparsers = chain_parser.add_subparsers(dest="chain_action", required=True)
|
|
581
|
+
|
|
582
|
+
chain_status_parser = chain_subparsers.add_parser("status", help="Get chain status")
|
|
583
|
+
chain_status_parser.add_argument("chain_id", help="Chain ID")
|
|
584
|
+
chain_status_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
585
|
+
chain_status_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
586
|
+
chain_status_parser.add_argument(
|
|
587
|
+
"--format", choices=["table", "json"], default="table"
|
|
588
|
+
)
|
|
589
|
+
chain_status_parser.set_defaults(func=cmd_chain_status)
|
|
590
|
+
|
|
591
|
+
chain_tasks_parser = chain_subparsers.add_parser(
|
|
592
|
+
"tasks", help="List tasks in a chain"
|
|
593
|
+
)
|
|
594
|
+
chain_tasks_parser.add_argument("chain_id", help="Chain ID")
|
|
595
|
+
chain_tasks_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
596
|
+
chain_tasks_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
597
|
+
chain_tasks_parser.add_argument(
|
|
598
|
+
"--format", choices=["table", "json"], default="table"
|
|
599
|
+
)
|
|
600
|
+
chain_tasks_parser.set_defaults(func=cmd_chain_tasks)
|
|
601
|
+
|
|
602
|
+
# ---------- health ----------
|
|
603
|
+
health_parser = subparsers.add_parser(
|
|
604
|
+
"health", help="Check system health (Redis connectivity)"
|
|
605
|
+
)
|
|
606
|
+
health_parser.add_argument("--redis-url", default=DEFAULT_REDIS_URL)
|
|
607
|
+
health_parser.add_argument("--queue-name", default=DEFAULT_QUEUE_NAME)
|
|
608
|
+
health_parser.add_argument("--task-prefix", default=DEFAULT_TASK_PREFIX)
|
|
609
|
+
health_parser.set_defaults(func=cmd_health)
|
|
610
|
+
|
|
611
|
+
return parser
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def main():
|
|
615
|
+
parser = create_parser()
|
|
616
|
+
args = parser.parse_args()
|
|
617
|
+
|
|
618
|
+
# Set logging based on global verbosity
|
|
619
|
+
setup_logging(args.verbose)
|
|
620
|
+
|
|
621
|
+
# Call the subcommand function
|
|
622
|
+
if hasattr(args, "func"):
|
|
623
|
+
try:
|
|
624
|
+
args.func(args)
|
|
625
|
+
except KeyboardInterrupt:
|
|
626
|
+
print("\nInterrupted", file=sys.stderr)
|
|
627
|
+
sys.exit(1)
|
|
628
|
+
except Exception as e:
|
|
629
|
+
logging.error(f"Error: {e}", exc_info=True)
|
|
630
|
+
sys.exit(1)
|
|
631
|
+
else:
|
|
632
|
+
parser.print_help()
|
|
633
|
+
sys.exit(1)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
if __name__ == "__main__":
|
|
637
|
+
main()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from broccoli.core.chain.task_chain import TaskChain
|
|
2
|
+
from broccoli.core.health import HealthCheck
|
|
3
|
+
from broccoli.core.result import ResultBackend
|
|
4
|
+
from broccoli.core.task.task import Task
|
|
5
|
+
from broccoli.core.task.task_queue import TaskQueue
|
|
6
|
+
from broccoli.core.task.task_registry import TaskRegistry
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"Task",
|
|
10
|
+
"TaskQueue",
|
|
11
|
+
"TaskRegistry",
|
|
12
|
+
"TaskChain",
|
|
13
|
+
"ResultBackend",
|
|
14
|
+
"HealthCheck",
|
|
15
|
+
]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import uuid
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class Chain:
|
|
9
|
+
chain_id: str(default_factory=lambda: str(uuid.uuid4()))
|
|
10
|
+
total_tasks: Optional[int] = None
|
|
11
|
+
completion_task: Optional[str] = None
|
|
12
|
+
status: str = "pending"
|
|
13
|
+
completed_tasks: int = 0
|
|
14
|
+
failed: bool = False
|
|
15
|
+
current_task: int = 0
|
|
16
|
+
result: any = None
|
|
17
|
+
|
|
18
|
+
def to_dict(self) -> dict:
|
|
19
|
+
return {
|
|
20
|
+
"chain_id": self.chain_id,
|
|
21
|
+
"completion_task": self.completion_task or "",
|
|
22
|
+
"status": self.status,
|
|
23
|
+
"completed_tasks": self.completed_tasks,
|
|
24
|
+
"failed": str(self.failed),
|
|
25
|
+
"total_tasks": str(self.total_tasks)
|
|
26
|
+
if self.total_tasks is not None
|
|
27
|
+
else None,
|
|
28
|
+
"current_task": self.current_task,
|
|
29
|
+
"result": json.dumps(self.result) if self.result is not None else None,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def from_dict(cls, data: dict) -> "Chain":
|
|
34
|
+
return cls(
|
|
35
|
+
chain_id=data.get("chain_id"),
|
|
36
|
+
completion_task=data.get("completion_task") or None,
|
|
37
|
+
status=data.get("status", "pending"),
|
|
38
|
+
completed_tasks=int(data.get("completed_tasks", 0)),
|
|
39
|
+
current_task=int(data.get("current_task", 0)),
|
|
40
|
+
result=data.get("result") if data.get("result") else None,
|
|
41
|
+
failed=data.get("failed", "False") == "True",
|
|
42
|
+
total_tasks=int(data.get("total_tasks", 0))
|
|
43
|
+
if data.get("total_tasks")
|
|
44
|
+
else None,
|
|
45
|
+
)
|