taskmanager-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskmanager/__init__.py +150 -0
- taskmanager/api/app.py +488 -0
- taskmanager/api/events.py +68 -0
- taskmanager/cli.py +291 -0
- taskmanager/config.py +19 -0
- taskmanager/contrib/__init__.py +3 -0
- taskmanager/contrib/django/__init__.py +6 -0
- taskmanager/contrib/django/apps.py +46 -0
- taskmanager/contrib/django/management/__init__.py +1 -0
- taskmanager/contrib/django/management/commands/__init__.py +1 -0
- taskmanager/contrib/django/management/commands/run_scheduler.py +44 -0
- taskmanager/contrib/django/management/commands/run_worker.py +72 -0
- taskmanager/contrib/django/urls.py +30 -0
- taskmanager/contrib/fastapi.py +31 -0
- taskmanager/core/broker.py +541 -0
- taskmanager/core/builtin_tasks.py +59 -0
- taskmanager/core/job.py +50 -0
- taskmanager/core/task.py +224 -0
- taskmanager/scheduler/cron.py +52 -0
- taskmanager/scheduler/scheduler.py +178 -0
- taskmanager/ui/app.js +1390 -0
- taskmanager/ui/index.html +681 -0
- taskmanager/ui/styles.css +1048 -0
- taskmanager/worker/heartbeat.py +150 -0
- taskmanager/worker/worker.py +207 -0
- taskmanager_engine-0.1.0.dist-info/METADATA +284 -0
- taskmanager_engine-0.1.0.dist-info/RECORD +30 -0
- taskmanager_engine-0.1.0.dist-info/WHEEL +5 -0
- taskmanager_engine-0.1.0.dist-info/entry_points.txt +2 -0
- taskmanager_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
import psutil
|
|
9
|
+
from pydantic import BaseModel, Field
|
|
10
|
+
|
|
11
|
+
from taskmanager.config import settings
|
|
12
|
+
from taskmanager.core.broker import RedisBroker
|
|
13
|
+
from taskmanager.core.job import JobStatus
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class WorkerInfo(BaseModel):
|
|
19
|
+
id: str
|
|
20
|
+
name: str
|
|
21
|
+
queues: list[str]
|
|
22
|
+
concurrency: int
|
|
23
|
+
status: str = "idle" # idle, busy, paused, stopped, dead
|
|
24
|
+
started_at: float = Field(default_factory=time.time)
|
|
25
|
+
last_heartbeat: float = Field(default_factory=time.time)
|
|
26
|
+
active_jobs_count: int = 0
|
|
27
|
+
completed_jobs_count: int = 0
|
|
28
|
+
failed_jobs_count: int = 0
|
|
29
|
+
cpu_percent: float = 0.0
|
|
30
|
+
memory_mb: float = 0.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class HeartbeatManager:
|
|
34
|
+
"""Manages worker vitality heartbeats and orphaned job reaping."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, broker: RedisBroker, worker_info: WorkerInfo):
|
|
37
|
+
self.broker = broker
|
|
38
|
+
self.worker_info = worker_info
|
|
39
|
+
self._running = False
|
|
40
|
+
self._task: asyncio.Task[None] | None = None
|
|
41
|
+
|
|
42
|
+
async def start(self) -> None:
|
|
43
|
+
self._running = True
|
|
44
|
+
try:
|
|
45
|
+
await self.send_heartbeat()
|
|
46
|
+
except Exception as err:
|
|
47
|
+
logger.warning(f"Initial heartbeat error: {err}")
|
|
48
|
+
self._task = asyncio.create_task(self._heartbeat_loop())
|
|
49
|
+
|
|
50
|
+
async def stop(self) -> None:
|
|
51
|
+
self._running = False
|
|
52
|
+
if self._task and not self._task.done():
|
|
53
|
+
self._task.cancel()
|
|
54
|
+
try:
|
|
55
|
+
await self._task
|
|
56
|
+
except asyncio.CancelledError:
|
|
57
|
+
pass
|
|
58
|
+
# Unregister worker on clean shutdown
|
|
59
|
+
await self.unregister_worker()
|
|
60
|
+
|
|
61
|
+
async def send_heartbeat(self) -> None:
|
|
62
|
+
"""Sends a single heartbeat ping updating Redis state with TTL."""
|
|
63
|
+
try:
|
|
64
|
+
process = psutil.Process(os.getpid())
|
|
65
|
+
mem_info = process.memory_info()
|
|
66
|
+
self.worker_info.memory_mb = round(mem_info.rss / (1024 * 1024), 2)
|
|
67
|
+
self.worker_info.cpu_percent = psutil.cpu_percent(interval=None)
|
|
68
|
+
except Exception:
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
self.worker_info.last_heartbeat = time.time()
|
|
72
|
+
key = self.broker._key_worker(self.worker_info.id)
|
|
73
|
+
data = self.worker_info.model_dump_json()
|
|
74
|
+
|
|
75
|
+
# Set worker info with TTL
|
|
76
|
+
await self.broker.redis.set(key, data, ex=int(settings.worker_heartbeat_ttl))
|
|
77
|
+
await self.broker.redis.sadd(self.broker._key_workers(), self.worker_info.id)
|
|
78
|
+
await self.broker.publish_event("worker:heartbeat", self.worker_info.model_dump())
|
|
79
|
+
|
|
80
|
+
async def unregister_worker(self) -> None:
|
|
81
|
+
"""Removes worker from registry upon shutdown."""
|
|
82
|
+
self.worker_info.status = "stopped"
|
|
83
|
+
key = self.broker._key_worker(self.worker_info.id)
|
|
84
|
+
await self.broker.redis.delete(key)
|
|
85
|
+
await self.broker.redis.srem(self.broker._key_workers(), self.worker_info.id)
|
|
86
|
+
await self.broker.publish_event("worker:stopped", {"worker_id": self.worker_info.id})
|
|
87
|
+
|
|
88
|
+
async def _heartbeat_loop(self) -> None:
|
|
89
|
+
while self._running:
|
|
90
|
+
try:
|
|
91
|
+
await self.send_heartbeat()
|
|
92
|
+
await asyncio.sleep(settings.worker_heartbeat_interval)
|
|
93
|
+
except asyncio.CancelledError:
|
|
94
|
+
break
|
|
95
|
+
except Exception as err:
|
|
96
|
+
logger.warning(f"Heartbeat failure for worker {self.worker_info.id}: {err}")
|
|
97
|
+
await asyncio.sleep(settings.worker_heartbeat_interval)
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
async def get_all_workers(cls, broker: RedisBroker) -> list[WorkerInfo]:
|
|
101
|
+
"""Returns all currently registered workers and cleans up expired ones."""
|
|
102
|
+
worker_ids = await broker.redis.smembers(broker._key_workers())
|
|
103
|
+
workers: list[WorkerInfo] = []
|
|
104
|
+
now = time.time()
|
|
105
|
+
|
|
106
|
+
for wid in list(worker_ids):
|
|
107
|
+
key = broker._key_worker(wid)
|
|
108
|
+
data = await broker.redis.get(key)
|
|
109
|
+
if data:
|
|
110
|
+
try:
|
|
111
|
+
info = WorkerInfo.model_validate_json(data)
|
|
112
|
+
if (now - info.last_heartbeat) > settings.worker_heartbeat_ttl:
|
|
113
|
+
info.status = "dead"
|
|
114
|
+
workers.append(info)
|
|
115
|
+
except Exception:
|
|
116
|
+
pass
|
|
117
|
+
else:
|
|
118
|
+
# Expired from TTL -> Clean up from workers set
|
|
119
|
+
await broker.redis.srem(broker._key_workers(), wid)
|
|
120
|
+
return workers
|
|
121
|
+
|
|
122
|
+
@classmethod
|
|
123
|
+
async def reap_orphans(cls, broker: RedisBroker) -> int:
|
|
124
|
+
"""Finds dead workers and recovers their un-acked active jobs back to pending."""
|
|
125
|
+
worker_ids = await broker.redis.smembers(broker._key_workers())
|
|
126
|
+
reaped_count = 0
|
|
127
|
+
|
|
128
|
+
for wid in worker_ids:
|
|
129
|
+
key = broker._key_worker(wid)
|
|
130
|
+
data = await broker.redis.get(key)
|
|
131
|
+
if not data:
|
|
132
|
+
# Worker TTL expired -> Reap active jobs
|
|
133
|
+
active_key = broker._key_active(wid)
|
|
134
|
+
orphaned_job_ids = await broker.redis.smembers(active_key)
|
|
135
|
+
for jid in orphaned_job_ids:
|
|
136
|
+
job = await broker.get_job(jid)
|
|
137
|
+
if job and job.status == JobStatus.ACTIVE:
|
|
138
|
+
logger.warning(f"Reaping orphaned job {jid} from dead worker {wid}")
|
|
139
|
+
job.status = JobStatus.PENDING
|
|
140
|
+
job.worker_id = None
|
|
141
|
+
await broker.save_job(job)
|
|
142
|
+
await broker.redis.rpush(broker._key_queue(job.queue), job.id)
|
|
143
|
+
await broker.publish_event(
|
|
144
|
+
"job:reaped", {"job_id": job.id, "worker_id": wid}
|
|
145
|
+
)
|
|
146
|
+
reaped_count += 1
|
|
147
|
+
# Clean active set & remove dead worker from set
|
|
148
|
+
await broker.redis.delete(active_key)
|
|
149
|
+
await broker.redis.srem(broker._key_workers(), wid)
|
|
150
|
+
return reaped_count
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import traceback
|
|
6
|
+
import uuid
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from taskmanager.core.broker import RedisBroker
|
|
10
|
+
from taskmanager.core.job import Job
|
|
11
|
+
from taskmanager.core.task import TaskRegistry, registry
|
|
12
|
+
from taskmanager.worker.heartbeat import HeartbeatManager, WorkerInfo
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Worker:
|
|
18
|
+
"""Asyncio background worker daemon capable of concurrent job execution."""
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
queues: list[str] | None = None,
|
|
23
|
+
concurrency: int = 5,
|
|
24
|
+
name: str | None = None,
|
|
25
|
+
max_memory_mb: float | None = None,
|
|
26
|
+
max_cpu_percent: float | None = None,
|
|
27
|
+
broker: RedisBroker | None = None,
|
|
28
|
+
task_registry: TaskRegistry | None = None,
|
|
29
|
+
):
|
|
30
|
+
self.id = str(uuid.uuid4())
|
|
31
|
+
self.name = name or f"worker-{self.id[:8]}"
|
|
32
|
+
self.queues = queues or ["default"]
|
|
33
|
+
self.concurrency = concurrency
|
|
34
|
+
self.max_memory_mb = max_memory_mb
|
|
35
|
+
self.max_cpu_percent = max_cpu_percent
|
|
36
|
+
self.registry = task_registry or registry
|
|
37
|
+
self.broker = broker or self.registry.get_broker()
|
|
38
|
+
self.semaphore = asyncio.Semaphore(concurrency)
|
|
39
|
+
|
|
40
|
+
self.info = WorkerInfo(
|
|
41
|
+
id=self.id,
|
|
42
|
+
name=self.name,
|
|
43
|
+
queues=self.queues,
|
|
44
|
+
concurrency=self.concurrency,
|
|
45
|
+
status="idle",
|
|
46
|
+
)
|
|
47
|
+
self.heartbeat = HeartbeatManager(self.broker, self.info)
|
|
48
|
+
self._running = False
|
|
49
|
+
self._paused = False
|
|
50
|
+
self._active_tasks: set[asyncio.Task[Any]] = set()
|
|
51
|
+
self._control_task: asyncio.Task[Any] | None = None
|
|
52
|
+
|
|
53
|
+
def pause(self) -> None:
|
|
54
|
+
"""Pauses job consumption."""
|
|
55
|
+
self._paused = True
|
|
56
|
+
self.info.status = "paused"
|
|
57
|
+
logger.info(f"Worker {self.name} paused.")
|
|
58
|
+
|
|
59
|
+
def resume(self) -> None:
|
|
60
|
+
"""Resumes job consumption."""
|
|
61
|
+
self._paused = False
|
|
62
|
+
self.info.status = "idle" if self.info.active_jobs_count == 0 else "busy"
|
|
63
|
+
logger.info(f"Worker {self.name} resumed.")
|
|
64
|
+
|
|
65
|
+
async def _listen_control(self) -> None:
|
|
66
|
+
"""Listens for remote control signals (pause, resume, stop) via Redis pub/sub."""
|
|
67
|
+
try:
|
|
68
|
+
async for cmd in self.broker.subscribe_control():
|
|
69
|
+
target_id = cmd.get("worker_id")
|
|
70
|
+
if target_id is None or target_id == self.id or target_id == self.name:
|
|
71
|
+
action = cmd.get("action")
|
|
72
|
+
if action == "pause":
|
|
73
|
+
self.pause()
|
|
74
|
+
elif action == "resume":
|
|
75
|
+
self.resume()
|
|
76
|
+
elif action == "stop":
|
|
77
|
+
asyncio.create_task(self.stop())
|
|
78
|
+
except asyncio.CancelledError:
|
|
79
|
+
pass
|
|
80
|
+
except Exception as err:
|
|
81
|
+
logger.debug(f"Worker control listener closed: {err}")
|
|
82
|
+
|
|
83
|
+
async def start(self) -> None:
|
|
84
|
+
"""Starts the worker processing loop, heartbeat manager, and control listener."""
|
|
85
|
+
self._running = True
|
|
86
|
+
self.info.status = "idle"
|
|
87
|
+
for q in self.queues:
|
|
88
|
+
await self.broker.redis.sadd(self.broker._key_queues(), q)
|
|
89
|
+
await self.heartbeat.start()
|
|
90
|
+
self._control_task = asyncio.create_task(self._listen_control())
|
|
91
|
+
limits_str = ""
|
|
92
|
+
if self.max_memory_mb or self.max_cpu_percent:
|
|
93
|
+
limits_str = f" [limits: memory={self.max_memory_mb or 'unlimited'}MB, cpu={self.max_cpu_percent or 'unlimited'}%]"
|
|
94
|
+
logger.info(
|
|
95
|
+
f"Worker {self.name} [{self.id}] started listening on {self.queues} (concurrency={self.concurrency}){limits_str}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
try:
|
|
99
|
+
while self._running:
|
|
100
|
+
if self._paused:
|
|
101
|
+
await asyncio.sleep(0.5)
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
# 1. Resource Backpressure / Guardrails check
|
|
105
|
+
if self.max_memory_mb and self.info.memory_mb >= self.max_memory_mb:
|
|
106
|
+
logger.warning(
|
|
107
|
+
f"⚠️ Worker {self.name} atingiu teto de memória ({self.info.memory_mb:.1f}MB >= {self.max_memory_mb}MB). Backpressure ativo (aguardando liberação de memória)..."
|
|
108
|
+
)
|
|
109
|
+
self.info.status = "throttled"
|
|
110
|
+
await asyncio.sleep(2.0)
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
if self.max_cpu_percent and self.info.cpu_percent >= self.max_cpu_percent:
|
|
114
|
+
logger.warning(
|
|
115
|
+
f"⚠️ Worker {self.name} atingiu teto de CPU ({self.info.cpu_percent:.1f}% >= {self.max_cpu_percent}%). Backpressure ativo..."
|
|
116
|
+
)
|
|
117
|
+
self.info.status = "throttled"
|
|
118
|
+
await asyncio.sleep(1.0)
|
|
119
|
+
continue
|
|
120
|
+
|
|
121
|
+
if self.info.status == "throttled":
|
|
122
|
+
self.info.status = "idle" if self.info.active_jobs_count == 0 else "busy"
|
|
123
|
+
|
|
124
|
+
# 2. Wait for available concurrency slot
|
|
125
|
+
await self.semaphore.acquire()
|
|
126
|
+
|
|
127
|
+
# 3. Fetch next job
|
|
128
|
+
job = await self.broker.fetch_next_job(self.queues, worker_id=self.id)
|
|
129
|
+
if job:
|
|
130
|
+
self.info.active_jobs_count += 1
|
|
131
|
+
self.info.status = "busy"
|
|
132
|
+
t = asyncio.create_task(self._process_job(job))
|
|
133
|
+
self._active_tasks.add(t)
|
|
134
|
+
t.add_done_callback(self._active_tasks.discard)
|
|
135
|
+
else:
|
|
136
|
+
# Release slot if no job was found
|
|
137
|
+
self.semaphore.release()
|
|
138
|
+
await asyncio.sleep(0.2)
|
|
139
|
+
except asyncio.CancelledError:
|
|
140
|
+
pass
|
|
141
|
+
finally:
|
|
142
|
+
await self.stop()
|
|
143
|
+
|
|
144
|
+
async def _process_job(self, job: Job) -> None:
|
|
145
|
+
"""Executes a single job with timeout handling and error catching."""
|
|
146
|
+
try:
|
|
147
|
+
task_def = self.registry.get(job.task_name)
|
|
148
|
+
if not task_def:
|
|
149
|
+
raise ValueError(
|
|
150
|
+
f"Task '{job.task_name}' is not registered in Worker task registry."
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# Execute with timeout if specified
|
|
154
|
+
timeout = job.timeout or task_def.timeout
|
|
155
|
+
if timeout and timeout > 0:
|
|
156
|
+
result = await asyncio.wait_for(task_def(*job.args, **job.kwargs), timeout=timeout)
|
|
157
|
+
else:
|
|
158
|
+
result = await task_def(*job.args, **job.kwargs)
|
|
159
|
+
|
|
160
|
+
await self.broker.mark_completed(job, result=result)
|
|
161
|
+
self.info.completed_jobs_count += 1
|
|
162
|
+
logger.info(f"Job {job.id} [{job.task_name}] completed successfully.")
|
|
163
|
+
|
|
164
|
+
except TimeoutError:
|
|
165
|
+
error_msg = f"Job timed out after {job.timeout or task_def.timeout}s"
|
|
166
|
+
tb_str = traceback.format_exc()
|
|
167
|
+
logger.error(f"Job {job.id} timed out: {error_msg}")
|
|
168
|
+
await self.broker.mark_failed(job, error=error_msg, traceback_str=tb_str)
|
|
169
|
+
self.info.failed_jobs_count += 1
|
|
170
|
+
|
|
171
|
+
except Exception as err:
|
|
172
|
+
error_msg = str(err) or type(err).__name__
|
|
173
|
+
tb_str = traceback.format_exc()
|
|
174
|
+
logger.error(f"Job {job.id} failed: {error_msg}")
|
|
175
|
+
await self.broker.mark_failed(job, error=error_msg, traceback_str=tb_str)
|
|
176
|
+
self.info.failed_jobs_count += 1
|
|
177
|
+
|
|
178
|
+
finally:
|
|
179
|
+
self.info.active_jobs_count = max(0, self.info.active_jobs_count - 1)
|
|
180
|
+
if self.info.active_jobs_count == 0 and not self._paused:
|
|
181
|
+
self.info.status = "idle"
|
|
182
|
+
self.semaphore.release()
|
|
183
|
+
|
|
184
|
+
async def stop(self, drain: bool = True, timeout: float = 10.0) -> None:
|
|
185
|
+
"""Stops the worker, optionally draining in-flight jobs."""
|
|
186
|
+
if not self._running and self.info.status == "stopped":
|
|
187
|
+
return
|
|
188
|
+
|
|
189
|
+
self._running = False
|
|
190
|
+
self.info.status = "stopped"
|
|
191
|
+
logger.info(f"Worker {self.name} stopping (drain={drain})...")
|
|
192
|
+
|
|
193
|
+
if self._control_task and not self._control_task.done():
|
|
194
|
+
self._control_task.cancel()
|
|
195
|
+
|
|
196
|
+
if drain and self._active_tasks:
|
|
197
|
+
try:
|
|
198
|
+
await asyncio.wait_for(
|
|
199
|
+
asyncio.gather(*list(self._active_tasks), return_exceptions=True),
|
|
200
|
+
timeout=timeout,
|
|
201
|
+
)
|
|
202
|
+
except TimeoutError:
|
|
203
|
+
logger.warning(f"Worker {self.name} drain timed out. Forcing shutdown.")
|
|
204
|
+
|
|
205
|
+
await self.heartbeat.stop()
|
|
206
|
+
logger.info(f"Worker {self.name} stopped.")
|
|
207
|
+
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: taskmanager-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance background task execution engine with dynamic cron scheduling and Linear-themed SPA dashboard
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: fastapi>=0.110.0
|
|
8
|
+
Requires-Dist: uvicorn[standard]>=0.28.0
|
|
9
|
+
Requires-Dist: redis[hiredis]>=5.0.0
|
|
10
|
+
Requires-Dist: pydantic>=2.6.0
|
|
11
|
+
Requires-Dist: croniter>=2.0.0
|
|
12
|
+
Requires-Dist: psutil>=5.9.0
|
|
13
|
+
Requires-Dist: fakeredis>=2.21.0
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
16
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
|
|
17
|
+
Requires-Dist: fakeredis>=2.21.0; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff>=0.3.0; extra == "dev"
|
|
19
|
+
Requires-Dist: httpx>=0.27.0; extra == "dev"
|
|
20
|
+
Requires-Dist: websockets>=12.0; extra == "dev"
|
|
21
|
+
|
|
22
|
+
# TaskManager ⚡
|
|
23
|
+
|
|
24
|
+
<div align="center">
|
|
25
|
+
|
|
26
|
+

|
|
27
|
+

|
|
28
|
+

|
|
29
|
+

|
|
30
|
+

|
|
31
|
+

|
|
32
|
+
|
|
33
|
+
**Engine moderna de execução e gerenciamento de background tasks em Python inspirada no Celery e BullMQ.**
|
|
34
|
+
*Agendador Cron Dinâmico • Observabilidade LGTM Completa • Dead Letter Queue Multi-Fila • Telemetria com Backpressure • Dashboard SPA Linear Dark.*
|
|
35
|
+
|
|
36
|
+
<br/>
|
|
37
|
+
|
|
38
|
+
<img src="docs/images/01_dashboard_overview.png" alt="TaskManager Dashboard Overview" width="100%" style="border-radius: 12px; border: 1px solid #23252a; box-shadow: 0 20px 40px rgba(0,0,0,0.6);" />
|
|
39
|
+
|
|
40
|
+
</div>
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 🌟 Principais Destaques
|
|
45
|
+
|
|
46
|
+
- **🚀 Asyncio & Sync Worker Runtime**: Suporte nativo e transparente para corrotinas assíncronas (`async def`) e funções síncronas (`def`), com concorrência ajustável e timeouts granulares.
|
|
47
|
+
- **📦 Redis Broker com Durabilidade AOF**: Sem brokers pesados. Utiliza listas atômicas do Redis (`LPOP`/`BLPOP`), conjuntos ordenados para jobs agendados e histórico, e persistência AOF para garantia de *Zero Data Loss*.
|
|
48
|
+
- **🧠 Fallback In-Memory Automático**: Modo de desenvolvimento zero-dependência (`fakeredis`) quando o Redis local não estiver em execução.
|
|
49
|
+
- **✨ Decorator `@task` & Introspecção**: Enfileiramento via `.delay(*args, **kwargs)` ou `.apply_async(...)` com geração automática de schemas e payload no painel.
|
|
50
|
+
- **⏰ Agendador Cron & Intervalos em Tempo Real**: Sintaxe padrão de 5 posições (`*/5 * * * *`) ou intervalo em segundos com distributed leader locking e execução garantida.
|
|
51
|
+
- **🛡️ Resiliência, DLQ & Backpressure**: Retentativas com exponential backoff, Dead Letter Queue (DLQ) com inspeção de stacktrace e *One-Click Replay*. Circuit breaker de CPU e Memória RSS por worker.
|
|
52
|
+
- **📊 Observabilidade LGTM Nativa**:
|
|
53
|
+
- **📈 Mimir / Prometheus**: KPIs em tempo real (Taxa de Sucesso %, Duração Média ms, Latência P95 ms, Throughput/min).
|
|
54
|
+
- **📜 Loki**: Console de logs de execução, erros e tracebacks capturados.
|
|
55
|
+
- **⏱️ Tempo**: Linha do tempo em cascata (Enqueued ➔ Dequeued ➔ Executing ➔ Finished/Failed).
|
|
56
|
+
- **🎨 Design System Linear Dark**: Interface minimalista com atalho global **`Ctrl+K` (Command Palette)**, menu de ação unificado **`+ Criar ▾`**, e realce suave de linhas.
|
|
57
|
+
- **🔌 Plug-and-Play em Qualquer Framework**: Use como aplicação independente ou embarque facilmente dentro do seu projeto **FastAPI**, **Django** ou **Flask**.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## 📦 Como Usar o TaskManager no seu Projeto (Biblioteca / Framework)
|
|
62
|
+
|
|
63
|
+
Você pode adicionar o TaskManager ao seu projeto Python existente de 3 maneiras:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
# Instalar no seu projeto:
|
|
67
|
+
pip install taskmanager
|
|
68
|
+
# ou usando UV:
|
|
69
|
+
uv add taskmanager
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
### Método 1: Embutir no FastAPI / Starlette (Sub-Aplicação / Mount)
|
|
75
|
+
|
|
76
|
+
Monte o dashboard e os endpoints do TaskManager diretamente dentro da sua aplicação FastAPI existente:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# main.py
|
|
80
|
+
from fastapi import FastAPI
|
|
81
|
+
from taskmanager import TaskManager, task
|
|
82
|
+
|
|
83
|
+
# 1. Cria sua aplicação principal
|
|
84
|
+
app = FastAPI(title="Minha API Principal")
|
|
85
|
+
|
|
86
|
+
# 2. Configura a instância do TaskManager apontando para o Redis
|
|
87
|
+
tm = TaskManager(redis_url="redis://localhost:6379/0", prefix="meu_app")
|
|
88
|
+
|
|
89
|
+
# 3. Define tarefas usando o decorator @task
|
|
90
|
+
@task(name="emails.enviar_boas_vindas", queue="emails", max_retries=3)
|
|
91
|
+
async def enviar_boas_vindas(email: str, nome: str):
|
|
92
|
+
print(f"Enviando e-mail para {nome} <{email}>...")
|
|
93
|
+
return {"status": "enviado"}
|
|
94
|
+
|
|
95
|
+
# 4. Monta o dashboard do TaskManager sob a rota /tasks
|
|
96
|
+
tm.mount_to(app, path="/tasks")
|
|
97
|
+
|
|
98
|
+
@app.post("/cadastro")
|
|
99
|
+
async def cadastrar_usuario(email: str, nome: str):
|
|
100
|
+
# Enfileira o job em background
|
|
101
|
+
job = await enviar_boas_vindas.delay(email=email, nome=nome)
|
|
102
|
+
return {"mensagem": "Usuário cadastrado!", "job_id": job.id}
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
*Ao acessar `http://localhost:8000/tasks/`, o dashboard completo estará rodando dentro da sua própria aplicação!*
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
### Método 2: Integração com Django (`taskmanager.contrib.django`)
|
|
110
|
+
|
|
111
|
+
O TaskManager possui integração de primeira classe com o ecossistema Django:
|
|
112
|
+
|
|
113
|
+
1. Adicione `'taskmanager.contrib.django'` ao seu `INSTALLED_APPS` em `settings.py`:
|
|
114
|
+
```python
|
|
115
|
+
# settings.py
|
|
116
|
+
INSTALLED_APPS = [
|
|
117
|
+
"django.contrib.admin",
|
|
118
|
+
"django.contrib.auth",
|
|
119
|
+
...,
|
|
120
|
+
"taskmanager.contrib.django", # <- Adicione aqui
|
|
121
|
+
"meu_app_vendas",
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
# Configurações opcionais do Redis
|
|
125
|
+
TASKMANAGER_REDIS_URL = "redis://localhost:6379/0"
|
|
126
|
+
TASKMANAGER_REDIS_PREFIX = "django_app"
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
2. Crie um arquivo `tasks.py` dentro dos seus apps Django:
|
|
130
|
+
```python
|
|
131
|
+
# meu_app_vendas/tasks.py
|
|
132
|
+
from taskmanager import task
|
|
133
|
+
|
|
134
|
+
@task(name="vendas.gerar_fatura", queue="faturamento", max_retries=2)
|
|
135
|
+
async def gerar_fatura(pedido_id: int):
|
|
136
|
+
# O taskmanager.contrib.django descobre e carrega automaticamente
|
|
137
|
+
# todos os módulos tasks.py de todos os apps em INSTALLED_APPS!
|
|
138
|
+
...
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
3. Adicione as URLs no seu `urls.py`:
|
|
142
|
+
```python
|
|
143
|
+
# urls.py
|
|
144
|
+
from django.urls import path, include
|
|
145
|
+
|
|
146
|
+
urlpatterns = [
|
|
147
|
+
path("admin/", admin.site.urls),
|
|
148
|
+
path("tasks/", include("taskmanager.contrib.django.urls")), # <- Dashboard
|
|
149
|
+
]
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
4. Execute workers e schedulers nativamente com `manage.py`:
|
|
153
|
+
```bash
|
|
154
|
+
# Iniciar worker consumindo as filas do Django:
|
|
155
|
+
python manage.py run_worker --queues faturamento,default --concurrency 5
|
|
156
|
+
|
|
157
|
+
# Iniciar o scheduler de rotinas cron:
|
|
158
|
+
python manage.py run_scheduler
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
### Método 3: Modo Standalone / CLI (Sidecar Desacoplado)
|
|
164
|
+
|
|
165
|
+
Se preferir manter o TaskManager como um serviço separado (sidecar):
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
# Modo Dev (Dashboard + Worker + Scheduler apontando para seus módulos de tarefas):
|
|
169
|
+
taskmanager dev --modules meu_projeto.tasks --port 8000
|
|
170
|
+
|
|
171
|
+
# Processos isolados para Produção:
|
|
172
|
+
taskmanager worker --modules meu_projeto.tasks --queues emails,default -c 8
|
|
173
|
+
taskmanager scheduler --modules meu_projeto.tasks
|
|
174
|
+
taskmanager server --port 8080
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## 📂 Pasta de Exemplos Práticos (`samples/`)
|
|
180
|
+
|
|
181
|
+
O repositório inclui projetos de exemplo completos e prontos para rodar:
|
|
182
|
+
|
|
183
|
+
| Exemplo | Descrição | Como Executar |
|
|
184
|
+
| :--- | :--- | :--- |
|
|
185
|
+
| [📁 samples/fastapi_sample](samples/fastapi_sample) | API FastAPI completa montando o TaskManager em `/tasks/` com endpoints de checkout e e-mail. | `uv run python -m samples.fastapi_sample.main` |
|
|
186
|
+
| [📁 samples/django_sample](samples/django_sample) | Projeto Django configurado com `taskmanager.contrib.django`, `tasks.py` e comandos `manage.py`. | `python samples/django_sample/manage.py run_worker` |
|
|
187
|
+
| [📁 samples/standalone_cli](samples/standalone_cli) | Demonstração de uso puro via linha de comando desacoplada. | `uv run taskmanager dev --modules samples.standalone_cli.tasks` |
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
## 🛠️ Como Compilar e Publicar a Biblioteca (PyPI)
|
|
192
|
+
|
|
193
|
+
Para gerar os pacotes `.whl` (Wheel) e `.tar.gz` (Source Distribution) com todos os arquivos de UI embutidos:
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
# 1. Compilar os artefatos de distribuição
|
|
197
|
+
uv build
|
|
198
|
+
# (ou via python -m build)
|
|
199
|
+
|
|
200
|
+
# 2. Testar instalação local em modo editável
|
|
201
|
+
pip install -e .
|
|
202
|
+
|
|
203
|
+
# 3. Publicar no PyPI
|
|
204
|
+
uv publish --token <SEU_TOKEN_PYPI>
|
|
205
|
+
# (ou twine upload dist/*)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 📸 Galeria de Telas do Dashboard
|
|
211
|
+
|
|
212
|
+
### 1. Visão Geral & Métricas em Tempo Real
|
|
213
|
+
Acompanhe o estado de todas as filas, consumo de hardware dos workers e log de eventos ao vivo via WebSockets.
|
|
214
|
+
<img src="docs/images/01_dashboard_overview.png" alt="Visão Geral" width="100%" style="border-radius: 8px; border: 1px solid #23252a; margin-bottom: 24px;" />
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
### 2. Menu Unificado `+ Criar ▾` & Paleta de Comandos (`Ctrl+K`)
|
|
219
|
+
Navegue e execute qualquer ação em milissegundos sem tirar as mãos do teclado.
|
|
220
|
+
<div align="center">
|
|
221
|
+
<img src="docs/images/08_quick_create_menu.png" alt="Menu Criar" width="48%" style="border-radius: 8px; border: 1px solid #23252a; margin-right: 2%;" />
|
|
222
|
+
<img src="docs/images/07_command_palette.png" alt="Command Palette Ctrl+K" width="48%" style="border-radius: 8px; border: 1px solid #23252a;" />
|
|
223
|
+
</div>
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
### 3. Gerenciamento de Workers & Proteção de Recursos
|
|
228
|
+
Monitore o uso de CPU/RAM de cada worker individual e pause ou interrompa processos com um clique.
|
|
229
|
+
<img src="docs/images/02_workers_management.png" alt="Gerenciamento de Workers" width="100%" style="border-radius: 8px; border: 1px solid #23252a; margin-bottom: 24px;" />
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
### 4. Agendamentos Cron & Rotinas Periódicas
|
|
234
|
+
Crie e edite agendamentos em tempo real sem precisar reiniciar os serviços ou fazer deploy.
|
|
235
|
+
<img src="docs/images/04_cron_schedules.png" alt="Agendamentos Cron" width="100%" style="border-radius: 8px; border: 1px solid #23252a; margin-bottom: 24px;" />
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
### 5. Dead Letter Queue (DLQ) & Inspeção de Falhas
|
|
240
|
+
Monitore jobs que esgotaram retentativas, visualize o stacktrace completo e faça o replay imediato para a fila.
|
|
241
|
+
<img src="docs/images/05_dlq_inspector.png" alt="Dead Letter Queue" width="100%" style="border-radius: 8px; border: 1px solid #23252a; margin-bottom: 24px;" />
|
|
242
|
+
|
|
243
|
+
---
|
|
244
|
+
|
|
245
|
+
### 6. Observabilidade & Trace Waterfall (LGTM Stack)
|
|
246
|
+
Inspecione a linha do tempo exata de execução de cada tarefa com logs capturados e payloads serializados.
|
|
247
|
+
<img src="docs/images/06_observability_trace.png" alt="Observabilidade e Tracing" width="100%" style="border-radius: 8px; border: 1px solid #23252a; margin-bottom: 24px;" />
|
|
248
|
+
|
|
249
|
+
---
|
|
250
|
+
|
|
251
|
+
## 🐳 Stack Docker & Docker Compose
|
|
252
|
+
|
|
253
|
+
Suba o cluster completo em contêineres com um único comando:
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
# Iniciar todos os serviços com Redis AOF persistente
|
|
257
|
+
docker compose up --build -d
|
|
258
|
+
|
|
259
|
+
# Visualizar status e healthchecks
|
|
260
|
+
docker compose ps
|
|
261
|
+
|
|
262
|
+
# Escalar workers dinamicamente
|
|
263
|
+
docker compose up -d --scale worker-emails=3
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
---
|
|
267
|
+
|
|
268
|
+
## 🧪 Testes & Qualidade
|
|
269
|
+
|
|
270
|
+
```bash
|
|
271
|
+
# Executar suíte completa de testes
|
|
272
|
+
uv run pytest -v tests/
|
|
273
|
+
|
|
274
|
+
# Executar linter de código
|
|
275
|
+
uv run ruff check taskmanager tests samples
|
|
276
|
+
|
|
277
|
+
# Sensor de Spec Drift (SDD)
|
|
278
|
+
node .agents/scripts/check-spec-drift.js
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
---
|
|
282
|
+
|
|
283
|
+
## 📄 Licença
|
|
284
|
+
Distribuído sob a licença [MIT](LICENSE). Pronto para uso individual ou corporativo.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
taskmanager/__init__.py,sha256=LsgOv2VOcBsLHnWzlEE9prFNqAtJLdCMggKyDDaktBg,4606
|
|
2
|
+
taskmanager/cli.py,sha256=lHUrhCxGUdg2uuXke0qXcdtApNDrpO2d8NBY9qPVq1U,11216
|
|
3
|
+
taskmanager/config.py,sha256=wvoRn2hq1JjvX0xlKRTX01H6lW182pcV6IolnsT8a5o,559
|
|
4
|
+
taskmanager/api/app.py,sha256=zrb-8XW8SfnCOz1IjdFMHk6YqsNe2sWFxE0L_XdNhu0,18886
|
|
5
|
+
taskmanager/api/events.py,sha256=FeUvUHqNTGCKjZJ04u7SLkfgnYzaAWmb3F94dYWckUQ,2253
|
|
6
|
+
taskmanager/contrib/__init__.py,sha256=9o6M3my91wl6rPXmerFS7sF7ENDjNHJoPJWsI7XqNIM,93
|
|
7
|
+
taskmanager/contrib/fastapi.py,sha256=SO2xtaWVRGV3KHcbLMfgMxp_CobQt5o6EdZPFSZq7ok,820
|
|
8
|
+
taskmanager/contrib/django/__init__.py,sha256=0BTjX3gIyArCz5QaZddPqIiCrAG38z92yQzbLQbHY5g,205
|
|
9
|
+
taskmanager/contrib/django/apps.py,sha256=6tQptFHxvR7L5u4FhSzIp-3U-FMYnN9DpGTfs0Bankk,1303
|
|
10
|
+
taskmanager/contrib/django/urls.py,sha256=BnxYCjsM7SnsJJRr2Dp3pa7z0s30ANrpAwkJEfPSF_E,778
|
|
11
|
+
taskmanager/contrib/django/management/__init__.py,sha256=XtGiUSnEbzJRa3QYOpERYL-doT2gPWt1BsJzLcjSdRo,48
|
|
12
|
+
taskmanager/contrib/django/management/commands/__init__.py,sha256=2lIYz8CyM8idv5jZovEFzEpngh9iuOqCuOeFsk8qLp0,62
|
|
13
|
+
taskmanager/contrib/django/management/commands/run_scheduler.py,sha256=Mox1KrycIwZTadBHVx7IbbgzgNc_NG_n1UJ-TNTqbek,1333
|
|
14
|
+
taskmanager/contrib/django/management/commands/run_worker.py,sha256=O44lu6e5QdAQrWt2IYzFj-fo_RbCJAlajVJp3BzvrNI,2294
|
|
15
|
+
taskmanager/core/broker.py,sha256=mS316ZX-6OlnOI6myNJ2-fVkJQ-BpFO1SqoEbONrmdQ,21069
|
|
16
|
+
taskmanager/core/builtin_tasks.py,sha256=pizSiIDUJHhAHmXYDLzhp2ahQ9YmubnOx9rYI-KOnVM,2067
|
|
17
|
+
taskmanager/core/job.py,sha256=CN7QMLOXFfI4YtwFGfWuZ9CXoa5wdXkcTmlyp42CioE,1536
|
|
18
|
+
taskmanager/core/task.py,sha256=s9D_0R2rIsP0qYCZdToxMLUiTtdCYvfXn8feTuhUK1I,8093
|
|
19
|
+
taskmanager/scheduler/cron.py,sha256=-JXljWCCupZtEXldOA_xfNAA2IpsNl0NspHdJMEMIkk,1909
|
|
20
|
+
taskmanager/scheduler/scheduler.py,sha256=dMfQXJp6lYw8IXCcslTcKQLZitTjshdt1vpXGDrz9Ew,6748
|
|
21
|
+
taskmanager/ui/app.js,sha256=8cpEkexz7skpByu2GoGuud0EWvI9CC-x0Hi-CiHBgjE,53217
|
|
22
|
+
taskmanager/ui/index.html,sha256=Vm7ZNswh72CFe5XIzB1aPZUVMUO1eZ5qUeLOtBZjLpQ,34016
|
|
23
|
+
taskmanager/ui/styles.css,sha256=bRVkbe9IhtIYcfEFckYrvp_BAR4V2gRBYaBEjZmO6Q0,19665
|
|
24
|
+
taskmanager/worker/heartbeat.py,sha256=h3KIQUhIjGZZrbydmP8YG-QPjIvaUgPyf4DGaKWPU7c,6086
|
|
25
|
+
taskmanager/worker/worker.py,sha256=2tggPDdtyjmRnTSa2oH1leXV-Yekp6ZLcx3rm2RGY0Y,8458
|
|
26
|
+
taskmanager_engine-0.1.0.dist-info/METADATA,sha256=g9wm6hT6hwhoq8Jqr7ltqR7SGJel5yZcdb-6tVN4Da4,11605
|
|
27
|
+
taskmanager_engine-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
28
|
+
taskmanager_engine-0.1.0.dist-info/entry_points.txt,sha256=Y0FYW_fak5JsupHPyp7E1WO0x5s_Mb40uM6G7UvIMqg,53
|
|
29
|
+
taskmanager_engine-0.1.0.dist-info/top_level.txt,sha256=DpRB7L9fWggaPMvAQLe6BpA4-ULwMkx0QUwasv9rN14,12
|
|
30
|
+
taskmanager_engine-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
taskmanager
|