codetether 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- a2a_server/__init__.py +29 -0
- a2a_server/a2a_agent_card.py +365 -0
- a2a_server/a2a_errors.py +1133 -0
- a2a_server/a2a_executor.py +926 -0
- a2a_server/a2a_router.py +1033 -0
- a2a_server/a2a_types.py +344 -0
- a2a_server/agent_card.py +408 -0
- a2a_server/agents_server.py +271 -0
- a2a_server/auth_api.py +349 -0
- a2a_server/billing_api.py +638 -0
- a2a_server/billing_service.py +712 -0
- a2a_server/billing_webhooks.py +501 -0
- a2a_server/config.py +96 -0
- a2a_server/database.py +2165 -0
- a2a_server/email_inbound.py +398 -0
- a2a_server/email_notifications.py +486 -0
- a2a_server/enhanced_agents.py +919 -0
- a2a_server/enhanced_server.py +160 -0
- a2a_server/hosted_worker.py +1049 -0
- a2a_server/integrated_agents_server.py +347 -0
- a2a_server/keycloak_auth.py +750 -0
- a2a_server/livekit_bridge.py +439 -0
- a2a_server/marketing_tools.py +1364 -0
- a2a_server/mcp_client.py +196 -0
- a2a_server/mcp_http_server.py +2256 -0
- a2a_server/mcp_server.py +191 -0
- a2a_server/message_broker.py +725 -0
- a2a_server/mock_mcp.py +273 -0
- a2a_server/models.py +494 -0
- a2a_server/monitor_api.py +5904 -0
- a2a_server/opencode_bridge.py +1594 -0
- a2a_server/redis_task_manager.py +518 -0
- a2a_server/server.py +726 -0
- a2a_server/task_manager.py +668 -0
- a2a_server/task_queue.py +742 -0
- a2a_server/tenant_api.py +333 -0
- a2a_server/tenant_middleware.py +219 -0
- a2a_server/tenant_service.py +760 -0
- a2a_server/user_auth.py +721 -0
- a2a_server/vault_client.py +576 -0
- a2a_server/worker_sse.py +873 -0
- agent_worker/__init__.py +8 -0
- agent_worker/worker.py +4877 -0
- codetether/__init__.py +10 -0
- codetether/__main__.py +4 -0
- codetether/cli.py +112 -0
- codetether/worker_cli.py +57 -0
- codetether-1.2.2.dist-info/METADATA +570 -0
- codetether-1.2.2.dist-info/RECORD +66 -0
- codetether-1.2.2.dist-info/WHEEL +5 -0
- codetether-1.2.2.dist-info/entry_points.txt +4 -0
- codetether-1.2.2.dist-info/licenses/LICENSE +202 -0
- codetether-1.2.2.dist-info/top_level.txt +5 -0
- codetether_voice_agent/__init__.py +6 -0
- codetether_voice_agent/agent.py +445 -0
- codetether_voice_agent/codetether_mcp.py +345 -0
- codetether_voice_agent/config.py +16 -0
- codetether_voice_agent/functiongemma_caller.py +380 -0
- codetether_voice_agent/session_playback.py +247 -0
- codetether_voice_agent/tools/__init__.py +21 -0
- codetether_voice_agent/tools/definitions.py +135 -0
- codetether_voice_agent/tools/handlers.py +380 -0
- run_server.py +314 -0
- ui/monitor-tailwind.html +1790 -0
- ui/monitor.html +1775 -0
- ui/monitor.js +2662 -0
a2a_server/task_queue.py
ADDED
|
@@ -0,0 +1,742 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Task Queue for Hosted Workers
|
|
3
|
+
|
|
4
|
+
This module provides the queue interface for the hosted worker system.
|
|
5
|
+
Tasks are enqueued when created and workers claim/execute them asynchronously.
|
|
6
|
+
|
|
7
|
+
Key concepts:
|
|
8
|
+
- task_runs table is the job queue (separate from tasks table)
|
|
9
|
+
- Jobs have leases that expire if workers die
|
|
10
|
+
- Per-user concurrency limits enforced at claim time
|
|
11
|
+
- Workers renew leases via heartbeat
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import logging
|
|
16
|
+
import uuid
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
from enum import Enum
|
|
20
|
+
from typing import Any, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class TaskRunStatus(str, Enum):
|
|
26
|
+
"""Status of a task run in the queue."""
|
|
27
|
+
|
|
28
|
+
QUEUED = 'queued'
|
|
29
|
+
RUNNING = 'running'
|
|
30
|
+
NEEDS_INPUT = 'needs_input'
|
|
31
|
+
COMPLETED = 'completed'
|
|
32
|
+
FAILED = 'failed'
|
|
33
|
+
CANCELLED = 'cancelled'
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class TaskLimitExceeded(Exception):
|
|
37
|
+
"""Raised when user has exceeded their task or concurrency limits."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
reason: str,
|
|
42
|
+
tasks_used: int = 0,
|
|
43
|
+
tasks_limit: int = 0,
|
|
44
|
+
running_count: int = 0,
|
|
45
|
+
concurrency_limit: int = 0,
|
|
46
|
+
):
|
|
47
|
+
self.reason = reason
|
|
48
|
+
self.tasks_used = tasks_used
|
|
49
|
+
self.tasks_limit = tasks_limit
|
|
50
|
+
self.running_count = running_count
|
|
51
|
+
self.concurrency_limit = concurrency_limit
|
|
52
|
+
super().__init__(reason)
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
55
|
+
"""Convert to dict for JSON response."""
|
|
56
|
+
return {
|
|
57
|
+
'error': 'task_limit_exceeded',
|
|
58
|
+
'message': self.reason,
|
|
59
|
+
'tasks_used': self.tasks_used,
|
|
60
|
+
'tasks_limit': self.tasks_limit,
|
|
61
|
+
'running_count': self.running_count,
|
|
62
|
+
'concurrency_limit': self.concurrency_limit,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class TaskRun:
|
|
68
|
+
"""A task execution in the queue."""
|
|
69
|
+
|
|
70
|
+
id: str
|
|
71
|
+
task_id: str
|
|
72
|
+
user_id: Optional[str] = None
|
|
73
|
+
template_id: Optional[str] = None
|
|
74
|
+
automation_id: Optional[str] = None
|
|
75
|
+
|
|
76
|
+
status: TaskRunStatus = TaskRunStatus.QUEUED
|
|
77
|
+
priority: int = 0
|
|
78
|
+
|
|
79
|
+
lease_owner: Optional[str] = None
|
|
80
|
+
lease_expires_at: Optional[datetime] = None
|
|
81
|
+
|
|
82
|
+
attempts: int = 0
|
|
83
|
+
max_attempts: int = 2
|
|
84
|
+
last_error: Optional[str] = None
|
|
85
|
+
|
|
86
|
+
started_at: Optional[datetime] = None
|
|
87
|
+
completed_at: Optional[datetime] = None
|
|
88
|
+
runtime_seconds: Optional[int] = None
|
|
89
|
+
|
|
90
|
+
result_summary: Optional[str] = None
|
|
91
|
+
result_full: Optional[Dict[str, Any]] = None
|
|
92
|
+
|
|
93
|
+
notify_email: Optional[str] = None
|
|
94
|
+
notify_webhook_url: Optional[str] = None
|
|
95
|
+
notification_sent: bool = False
|
|
96
|
+
|
|
97
|
+
# Agent routing fields (Phase 2 - agent-targeted routing)
|
|
98
|
+
target_agent_name: Optional[str] = None # If set, only this agent can claim
|
|
99
|
+
required_capabilities: Optional[List[str]] = None # Worker must have ALL
|
|
100
|
+
deadline_at: Optional[datetime] = None # Fail if not claimed by this time
|
|
101
|
+
routing_failed_at: Optional[datetime] = None
|
|
102
|
+
routing_failure_reason: Optional[str] = None
|
|
103
|
+
|
|
104
|
+
created_at: datetime = field(
|
|
105
|
+
default_factory=lambda: datetime.now(timezone.utc)
|
|
106
|
+
)
|
|
107
|
+
updated_at: datetime = field(
|
|
108
|
+
default_factory=lambda: datetime.now(timezone.utc)
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class TaskQueue:
|
|
113
|
+
"""
|
|
114
|
+
Interface to the task_runs queue.
|
|
115
|
+
|
|
116
|
+
This is the main API for enqueuing tasks and checking queue status.
|
|
117
|
+
Workers use the claim_* and complete_* functions directly via SQL.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
def __init__(self, db_pool):
|
|
121
|
+
"""
|
|
122
|
+
Initialize the task queue.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
db_pool: asyncpg connection pool
|
|
126
|
+
"""
|
|
127
|
+
self._pool = db_pool
|
|
128
|
+
|
|
129
|
+
async def enqueue(
|
|
130
|
+
self,
|
|
131
|
+
task_id: str,
|
|
132
|
+
user_id: Optional[str] = None,
|
|
133
|
+
template_id: Optional[str] = None,
|
|
134
|
+
automation_id: Optional[str] = None,
|
|
135
|
+
priority: int = 0,
|
|
136
|
+
notify_email: Optional[str] = None,
|
|
137
|
+
notify_webhook_url: Optional[str] = None,
|
|
138
|
+
skip_limit_check: bool = False,
|
|
139
|
+
# Agent routing parameters
|
|
140
|
+
target_agent_name: Optional[str] = None,
|
|
141
|
+
required_capabilities: Optional[List[str]] = None,
|
|
142
|
+
deadline_at: Optional[datetime] = None,
|
|
143
|
+
) -> TaskRun:
|
|
144
|
+
"""
|
|
145
|
+
Enqueue a task for execution by hosted workers.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
task_id: ID of the task to execute
|
|
149
|
+
user_id: Owner user ID (for concurrency limiting)
|
|
150
|
+
template_id: Template that generated this task (optional)
|
|
151
|
+
automation_id: Automation that generated this task (optional)
|
|
152
|
+
priority: Higher = more urgent (default 0)
|
|
153
|
+
notify_email: Email to notify on completion
|
|
154
|
+
notify_webhook_url: Webhook to call on completion
|
|
155
|
+
skip_limit_check: Skip limit enforcement (for internal/admin use)
|
|
156
|
+
target_agent_name: If set, only this agent can claim the task
|
|
157
|
+
required_capabilities: List of capabilities the worker must have
|
|
158
|
+
deadline_at: If set, task fails if not claimed by this time
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
TaskRun object representing the queued job
|
|
162
|
+
|
|
163
|
+
Raises:
|
|
164
|
+
TaskLimitExceeded: If user has exceeded their task or concurrency limits
|
|
165
|
+
"""
|
|
166
|
+
import json as json_module
|
|
167
|
+
|
|
168
|
+
run_id = str(uuid.uuid4())
|
|
169
|
+
|
|
170
|
+
async with self._pool.acquire() as conn:
|
|
171
|
+
# Check user limits before enqueuing (unless skipped)
|
|
172
|
+
if user_id and not skip_limit_check:
|
|
173
|
+
limit_check = await conn.fetchrow(
|
|
174
|
+
'SELECT * FROM check_user_task_limits($1)', user_id
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if limit_check and not limit_check['allowed']:
|
|
178
|
+
raise TaskLimitExceeded(
|
|
179
|
+
reason=limit_check['reason'],
|
|
180
|
+
tasks_used=limit_check['tasks_used'],
|
|
181
|
+
tasks_limit=limit_check['tasks_limit'],
|
|
182
|
+
running_count=limit_check['running_count'],
|
|
183
|
+
concurrency_limit=limit_check['concurrency_limit'],
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# Convert capabilities list to JSON for storage
|
|
187
|
+
capabilities_json = (
|
|
188
|
+
json_module.dumps(required_capabilities)
|
|
189
|
+
if required_capabilities
|
|
190
|
+
else None
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
# Enqueue the task with routing fields
|
|
194
|
+
await conn.execute(
|
|
195
|
+
"""
|
|
196
|
+
INSERT INTO task_runs (
|
|
197
|
+
id, task_id, user_id, template_id, automation_id,
|
|
198
|
+
status, priority, notify_email, notify_webhook_url,
|
|
199
|
+
target_agent_name, required_capabilities, deadline_at,
|
|
200
|
+
created_at, updated_at
|
|
201
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $13)
|
|
202
|
+
""",
|
|
203
|
+
run_id,
|
|
204
|
+
task_id,
|
|
205
|
+
user_id,
|
|
206
|
+
template_id,
|
|
207
|
+
automation_id,
|
|
208
|
+
TaskRunStatus.QUEUED.value,
|
|
209
|
+
priority,
|
|
210
|
+
notify_email,
|
|
211
|
+
notify_webhook_url,
|
|
212
|
+
target_agent_name,
|
|
213
|
+
capabilities_json,
|
|
214
|
+
deadline_at,
|
|
215
|
+
datetime.now(timezone.utc),
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
# Increment user's task usage counter
|
|
219
|
+
if user_id:
|
|
220
|
+
await conn.execute(
|
|
221
|
+
"""
|
|
222
|
+
UPDATE users
|
|
223
|
+
SET tasks_used_this_month = tasks_used_this_month + 1,
|
|
224
|
+
updated_at = NOW()
|
|
225
|
+
WHERE id = $1
|
|
226
|
+
""",
|
|
227
|
+
user_id,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
# Build log message with routing info
|
|
231
|
+
routing_info = ''
|
|
232
|
+
if target_agent_name:
|
|
233
|
+
routing_info += f', target_agent={target_agent_name}'
|
|
234
|
+
if deadline_at:
|
|
235
|
+
routing_info += f', deadline={deadline_at.isoformat()}'
|
|
236
|
+
|
|
237
|
+
logger.info(
|
|
238
|
+
f'Enqueued task run {run_id} for task {task_id} '
|
|
239
|
+
f'(user={user_id}, priority={priority}{routing_info})'
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return TaskRun(
|
|
243
|
+
id=run_id,
|
|
244
|
+
task_id=task_id,
|
|
245
|
+
user_id=user_id,
|
|
246
|
+
template_id=template_id,
|
|
247
|
+
automation_id=automation_id,
|
|
248
|
+
priority=priority,
|
|
249
|
+
notify_email=notify_email,
|
|
250
|
+
notify_webhook_url=notify_webhook_url,
|
|
251
|
+
target_agent_name=target_agent_name,
|
|
252
|
+
required_capabilities=required_capabilities,
|
|
253
|
+
deadline_at=deadline_at,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
async def get_run(self, run_id: str) -> Optional[TaskRun]:
|
|
257
|
+
"""Get a task run by ID."""
|
|
258
|
+
async with self._pool.acquire() as conn:
|
|
259
|
+
row = await conn.fetchrow(
|
|
260
|
+
'SELECT * FROM task_runs WHERE id = $1', run_id
|
|
261
|
+
)
|
|
262
|
+
if row:
|
|
263
|
+
return self._row_to_task_run(row)
|
|
264
|
+
return None
|
|
265
|
+
|
|
266
|
+
async def get_run_by_task(self, task_id: str) -> Optional[TaskRun]:
|
|
267
|
+
"""Get the most recent task run for a task."""
|
|
268
|
+
async with self._pool.acquire() as conn:
|
|
269
|
+
row = await conn.fetchrow(
|
|
270
|
+
"""
|
|
271
|
+
SELECT * FROM task_runs
|
|
272
|
+
WHERE task_id = $1
|
|
273
|
+
ORDER BY created_at DESC
|
|
274
|
+
LIMIT 1
|
|
275
|
+
""",
|
|
276
|
+
task_id,
|
|
277
|
+
)
|
|
278
|
+
if row:
|
|
279
|
+
return self._row_to_task_run(row)
|
|
280
|
+
return None
|
|
281
|
+
|
|
282
|
+
async def list_runs(
|
|
283
|
+
self,
|
|
284
|
+
user_id: Optional[str] = None,
|
|
285
|
+
status: Optional[TaskRunStatus] = None,
|
|
286
|
+
limit: int = 100,
|
|
287
|
+
) -> List[TaskRun]:
|
|
288
|
+
"""List task runs with optional filtering."""
|
|
289
|
+
conditions = []
|
|
290
|
+
params = []
|
|
291
|
+
param_idx = 1
|
|
292
|
+
|
|
293
|
+
if user_id:
|
|
294
|
+
conditions.append(f'user_id = ${param_idx}')
|
|
295
|
+
params.append(user_id)
|
|
296
|
+
param_idx += 1
|
|
297
|
+
|
|
298
|
+
if status:
|
|
299
|
+
conditions.append(f'status = ${param_idx}')
|
|
300
|
+
params.append(status.value)
|
|
301
|
+
param_idx += 1
|
|
302
|
+
|
|
303
|
+
where_clause = ' AND '.join(conditions) if conditions else 'TRUE'
|
|
304
|
+
|
|
305
|
+
async with self._pool.acquire() as conn:
|
|
306
|
+
rows = await conn.fetch(
|
|
307
|
+
f"""
|
|
308
|
+
SELECT * FROM task_runs
|
|
309
|
+
WHERE {where_clause}
|
|
310
|
+
ORDER BY created_at DESC
|
|
311
|
+
LIMIT ${param_idx}
|
|
312
|
+
""",
|
|
313
|
+
*params,
|
|
314
|
+
limit,
|
|
315
|
+
)
|
|
316
|
+
return [self._row_to_task_run(row) for row in rows]
|
|
317
|
+
|
|
318
|
+
async def get_queue_stats(self) -> Dict[str, Any]:
|
|
319
|
+
"""Get queue statistics."""
|
|
320
|
+
async with self._pool.acquire() as conn:
|
|
321
|
+
# Overall stats
|
|
322
|
+
stats = await conn.fetchrow(
|
|
323
|
+
"""
|
|
324
|
+
SELECT
|
|
325
|
+
COUNT(*) FILTER (WHERE status = 'queued') as queued,
|
|
326
|
+
COUNT(*) FILTER (WHERE status = 'running') as running,
|
|
327
|
+
COUNT(*) FILTER (WHERE status = 'completed') as completed_24h,
|
|
328
|
+
COUNT(*) FILTER (WHERE status = 'failed') as failed_24h,
|
|
329
|
+
AVG(runtime_seconds) FILTER (WHERE status = 'completed') as avg_runtime,
|
|
330
|
+
AVG(EXTRACT(EPOCH FROM (NOW() - created_at)))
|
|
331
|
+
FILTER (WHERE status = 'queued') as avg_wait_seconds
|
|
332
|
+
FROM task_runs
|
|
333
|
+
WHERE created_at > NOW() - INTERVAL '24 hours'
|
|
334
|
+
"""
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
# Per-user running counts
|
|
338
|
+
user_running = await conn.fetch(
|
|
339
|
+
"""
|
|
340
|
+
SELECT user_id, COUNT(*) as running_count
|
|
341
|
+
FROM task_runs
|
|
342
|
+
WHERE status = 'running'
|
|
343
|
+
GROUP BY user_id
|
|
344
|
+
"""
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
return {
|
|
348
|
+
'queued': stats['queued'] or 0,
|
|
349
|
+
'running': stats['running'] or 0,
|
|
350
|
+
'completed_24h': stats['completed_24h'] or 0,
|
|
351
|
+
'failed_24h': stats['failed_24h'] or 0,
|
|
352
|
+
'avg_runtime_seconds': float(stats['avg_runtime'] or 0),
|
|
353
|
+
'avg_wait_seconds': float(stats['avg_wait_seconds'] or 0),
|
|
354
|
+
'users_with_running_tasks': len(user_running),
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async def get_full_queue_status(self) -> Dict[str, Any]:
|
|
358
|
+
"""
|
|
359
|
+
Get comprehensive queue status for admin dashboard.
|
|
360
|
+
|
|
361
|
+
Returns queue counts, notification statuses, and worker health.
|
|
362
|
+
"""
|
|
363
|
+
async with self._pool.acquire() as conn:
|
|
364
|
+
# Queue counts by status
|
|
365
|
+
queue_stats = await conn.fetchrow(
|
|
366
|
+
"""
|
|
367
|
+
SELECT
|
|
368
|
+
COUNT(*) FILTER (WHERE status = 'queued') as queued,
|
|
369
|
+
COUNT(*) FILTER (WHERE status = 'running') as running,
|
|
370
|
+
COUNT(*) FILTER (WHERE status = 'needs_input') as needs_input,
|
|
371
|
+
COUNT(*) FILTER (WHERE status = 'completed' AND created_at > NOW() - INTERVAL '24 hours') as completed_24h,
|
|
372
|
+
COUNT(*) FILTER (WHERE status = 'failed' AND created_at > NOW() - INTERVAL '24 hours') as failed_24h,
|
|
373
|
+
AVG(EXTRACT(EPOCH FROM (NOW() - created_at)))
|
|
374
|
+
FILTER (WHERE status = 'queued') as avg_queue_wait_seconds,
|
|
375
|
+
MAX(EXTRACT(EPOCH FROM (NOW() - created_at)))
|
|
376
|
+
FILTER (WHERE status = 'queued') as max_queue_wait_seconds
|
|
377
|
+
FROM task_runs
|
|
378
|
+
"""
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
# Notification stats
|
|
382
|
+
notification_stats = await conn.fetchrow(
|
|
383
|
+
"""
|
|
384
|
+
SELECT
|
|
385
|
+
COUNT(*) FILTER (WHERE notification_status = 'failed' AND notification_next_retry_at <= NOW()) as email_failed_ready,
|
|
386
|
+
COUNT(*) FILTER (WHERE notification_status = 'pending' AND updated_at < NOW() - INTERVAL '5 minutes') as email_pending_stuck,
|
|
387
|
+
COUNT(*) FILTER (WHERE webhook_status = 'failed' AND webhook_next_retry_at <= NOW()) as webhook_failed_ready,
|
|
388
|
+
COUNT(*) FILTER (WHERE webhook_status = 'pending' AND updated_at < NOW() - INTERVAL '5 minutes') as webhook_pending_stuck,
|
|
389
|
+
COUNT(*) FILTER (WHERE notification_status = 'sent') as emails_sent_total,
|
|
390
|
+
COUNT(*) FILTER (WHERE webhook_status = 'sent') as webhooks_sent_total
|
|
391
|
+
FROM task_runs
|
|
392
|
+
WHERE created_at > NOW() - INTERVAL '24 hours'
|
|
393
|
+
"""
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
# Worker stats
|
|
397
|
+
worker_stats = await conn.fetchrow(
|
|
398
|
+
"""
|
|
399
|
+
SELECT
|
|
400
|
+
COUNT(*) FILTER (WHERE status = 'active') as active_pools,
|
|
401
|
+
SUM(max_concurrent_tasks) FILTER (WHERE status = 'active') as total_capacity,
|
|
402
|
+
SUM(current_tasks) FILTER (WHERE status = 'active') as current_load,
|
|
403
|
+
MAX(last_heartbeat) as last_heartbeat
|
|
404
|
+
FROM hosted_workers
|
|
405
|
+
"""
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
return {
|
|
409
|
+
'queue': {
|
|
410
|
+
'queued': queue_stats['queued'] or 0,
|
|
411
|
+
'running': queue_stats['running'] or 0,
|
|
412
|
+
'needs_input': queue_stats['needs_input'] or 0,
|
|
413
|
+
'completed_24h': queue_stats['completed_24h'] or 0,
|
|
414
|
+
'failed_24h': queue_stats['failed_24h'] or 0,
|
|
415
|
+
'avg_wait_seconds': round(
|
|
416
|
+
float(queue_stats['avg_queue_wait_seconds'] or 0), 1
|
|
417
|
+
),
|
|
418
|
+
'max_wait_seconds': int(
|
|
419
|
+
queue_stats['max_queue_wait_seconds'] or 0
|
|
420
|
+
),
|
|
421
|
+
},
|
|
422
|
+
'notifications': {
|
|
423
|
+
'email_failed_ready': notification_stats[
|
|
424
|
+
'email_failed_ready'
|
|
425
|
+
]
|
|
426
|
+
or 0,
|
|
427
|
+
'email_pending_stuck': notification_stats[
|
|
428
|
+
'email_pending_stuck'
|
|
429
|
+
]
|
|
430
|
+
or 0,
|
|
431
|
+
'webhook_failed_ready': notification_stats[
|
|
432
|
+
'webhook_failed_ready'
|
|
433
|
+
]
|
|
434
|
+
or 0,
|
|
435
|
+
'webhook_pending_stuck': notification_stats[
|
|
436
|
+
'webhook_pending_stuck'
|
|
437
|
+
]
|
|
438
|
+
or 0,
|
|
439
|
+
'emails_sent_24h': notification_stats['emails_sent_total']
|
|
440
|
+
or 0,
|
|
441
|
+
'webhooks_sent_24h': notification_stats[
|
|
442
|
+
'webhooks_sent_total'
|
|
443
|
+
]
|
|
444
|
+
or 0,
|
|
445
|
+
},
|
|
446
|
+
'workers': {
|
|
447
|
+
'active_pools': worker_stats['active_pools'] or 0,
|
|
448
|
+
'total_capacity': int(worker_stats['total_capacity'] or 0),
|
|
449
|
+
'current_load': int(worker_stats['current_load'] or 0),
|
|
450
|
+
'last_heartbeat': worker_stats['last_heartbeat'].isoformat()
|
|
451
|
+
if worker_stats['last_heartbeat']
|
|
452
|
+
else None,
|
|
453
|
+
},
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async def get_user_queue_status(self, user_id: str) -> Dict[str, Any]:
|
|
457
|
+
"""
|
|
458
|
+
Get queue status scoped to a specific user.
|
|
459
|
+
|
|
460
|
+
Returns the user's queued/running counts, recent task history,
|
|
461
|
+
and active runs with notification status.
|
|
462
|
+
"""
|
|
463
|
+
async with self._pool.acquire() as conn:
|
|
464
|
+
# User's queue counts
|
|
465
|
+
queue_stats = await conn.fetchrow(
|
|
466
|
+
"""
|
|
467
|
+
SELECT
|
|
468
|
+
COUNT(*) FILTER (WHERE status = 'queued') as queued,
|
|
469
|
+
COUNT(*) FILTER (WHERE status = 'running') as running,
|
|
470
|
+
COUNT(*) FILTER (WHERE status = 'needs_input') as needs_input,
|
|
471
|
+
COUNT(*) FILTER (WHERE status = 'completed' AND created_at > NOW() - INTERVAL '24 hours') as completed_24h,
|
|
472
|
+
COUNT(*) FILTER (WHERE status = 'failed' AND created_at > NOW() - INTERVAL '24 hours') as failed_24h,
|
|
473
|
+
COUNT(*) FILTER (WHERE created_at > NOW() - INTERVAL '30 days') as total_this_month
|
|
474
|
+
FROM task_runs
|
|
475
|
+
WHERE user_id = $1
|
|
476
|
+
""",
|
|
477
|
+
user_id,
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
# User's active runs (queued or running)
|
|
481
|
+
active_runs = await conn.fetch(
|
|
482
|
+
"""
|
|
483
|
+
SELECT
|
|
484
|
+
tr.id,
|
|
485
|
+
tr.task_id,
|
|
486
|
+
tr.status,
|
|
487
|
+
tr.priority,
|
|
488
|
+
tr.started_at,
|
|
489
|
+
tr.created_at,
|
|
490
|
+
tr.runtime_seconds,
|
|
491
|
+
tr.notification_status,
|
|
492
|
+
tr.result_summary,
|
|
493
|
+
t.title
|
|
494
|
+
FROM task_runs tr
|
|
495
|
+
LEFT JOIN tasks t ON tr.task_id = t.id
|
|
496
|
+
WHERE tr.user_id = $1
|
|
497
|
+
AND tr.status IN ('queued', 'running', 'needs_input')
|
|
498
|
+
ORDER BY
|
|
499
|
+
CASE tr.status WHEN 'running' THEN 0 WHEN 'needs_input' THEN 1 ELSE 2 END,
|
|
500
|
+
tr.priority DESC,
|
|
501
|
+
tr.created_at ASC
|
|
502
|
+
LIMIT 20
|
|
503
|
+
""",
|
|
504
|
+
user_id,
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# User's recent completed/failed runs
|
|
508
|
+
recent_runs = await conn.fetch(
|
|
509
|
+
"""
|
|
510
|
+
SELECT
|
|
511
|
+
tr.id,
|
|
512
|
+
tr.task_id,
|
|
513
|
+
tr.status,
|
|
514
|
+
tr.completed_at,
|
|
515
|
+
tr.runtime_seconds,
|
|
516
|
+
tr.notification_status,
|
|
517
|
+
tr.result_summary,
|
|
518
|
+
t.title
|
|
519
|
+
FROM task_runs tr
|
|
520
|
+
LEFT JOIN tasks t ON tr.task_id = t.id
|
|
521
|
+
WHERE tr.user_id = $1
|
|
522
|
+
AND tr.status IN ('completed', 'failed')
|
|
523
|
+
AND tr.created_at > NOW() - INTERVAL '24 hours'
|
|
524
|
+
ORDER BY tr.completed_at DESC
|
|
525
|
+
LIMIT 10
|
|
526
|
+
""",
|
|
527
|
+
user_id,
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
# User's limits (from users table)
|
|
531
|
+
user_limits = await conn.fetchrow(
|
|
532
|
+
"""
|
|
533
|
+
SELECT
|
|
534
|
+
concurrency_limit,
|
|
535
|
+
tasks_limit,
|
|
536
|
+
tasks_used_this_month,
|
|
537
|
+
max_runtime_seconds,
|
|
538
|
+
tier_id
|
|
539
|
+
FROM users
|
|
540
|
+
WHERE id = $1
|
|
541
|
+
""",
|
|
542
|
+
user_id,
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
return {
|
|
546
|
+
'queue': {
|
|
547
|
+
'queued': queue_stats['queued'] or 0,
|
|
548
|
+
'running': queue_stats['running'] or 0,
|
|
549
|
+
'needs_input': queue_stats['needs_input'] or 0,
|
|
550
|
+
'completed_24h': queue_stats['completed_24h'] or 0,
|
|
551
|
+
'failed_24h': queue_stats['failed_24h'] or 0,
|
|
552
|
+
'total_this_month': queue_stats['total_this_month'] or 0,
|
|
553
|
+
},
|
|
554
|
+
'limits': {
|
|
555
|
+
'concurrency_limit': user_limits['concurrency_limit']
|
|
556
|
+
if user_limits
|
|
557
|
+
else 1,
|
|
558
|
+
'tasks_limit': user_limits['tasks_limit']
|
|
559
|
+
if user_limits
|
|
560
|
+
else 10,
|
|
561
|
+
'tasks_used': user_limits['tasks_used_this_month']
|
|
562
|
+
if user_limits
|
|
563
|
+
else 0,
|
|
564
|
+
'max_runtime_seconds': user_limits['max_runtime_seconds']
|
|
565
|
+
if user_limits
|
|
566
|
+
else 600,
|
|
567
|
+
'tier': user_limits['tier_id'] if user_limits else 'free',
|
|
568
|
+
},
|
|
569
|
+
'active_runs': [
|
|
570
|
+
{
|
|
571
|
+
'id': run['id'],
|
|
572
|
+
'task_id': run['task_id'],
|
|
573
|
+
'title': run['title'] or 'Untitled',
|
|
574
|
+
'status': run['status'],
|
|
575
|
+
'priority': run['priority'],
|
|
576
|
+
'started_at': run['started_at'].isoformat()
|
|
577
|
+
if run['started_at']
|
|
578
|
+
else None,
|
|
579
|
+
'created_at': run['created_at'].isoformat()
|
|
580
|
+
if run['created_at']
|
|
581
|
+
else None,
|
|
582
|
+
'runtime_seconds': run['runtime_seconds'],
|
|
583
|
+
'notification_status': run['notification_status'],
|
|
584
|
+
}
|
|
585
|
+
for run in active_runs
|
|
586
|
+
],
|
|
587
|
+
'recent_runs': [
|
|
588
|
+
{
|
|
589
|
+
'id': run['id'],
|
|
590
|
+
'task_id': run['task_id'],
|
|
591
|
+
'title': run['title'] or 'Untitled',
|
|
592
|
+
'status': run['status'],
|
|
593
|
+
'completed_at': run['completed_at'].isoformat()
|
|
594
|
+
if run['completed_at']
|
|
595
|
+
else None,
|
|
596
|
+
'runtime_seconds': run['runtime_seconds'],
|
|
597
|
+
'notification_status': run['notification_status'],
|
|
598
|
+
'result_summary': run['result_summary'],
|
|
599
|
+
}
|
|
600
|
+
for run in recent_runs
|
|
601
|
+
],
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
async def cancel_run(self, run_id: str) -> bool:
|
|
605
|
+
"""Cancel a queued task run."""
|
|
606
|
+
async with self._pool.acquire() as conn:
|
|
607
|
+
result = await conn.execute(
|
|
608
|
+
"""
|
|
609
|
+
UPDATE task_runs SET
|
|
610
|
+
status = 'cancelled',
|
|
611
|
+
updated_at = NOW()
|
|
612
|
+
WHERE id = $1 AND status = 'queued'
|
|
613
|
+
""",
|
|
614
|
+
run_id,
|
|
615
|
+
)
|
|
616
|
+
return result == 'UPDATE 1'
|
|
617
|
+
|
|
618
|
+
async def reclaim_expired_leases(self) -> int:
|
|
619
|
+
"""
|
|
620
|
+
Reclaim jobs with expired leases.
|
|
621
|
+
|
|
622
|
+
Should be called periodically (e.g., every 30 seconds).
|
|
623
|
+
Returns number of jobs reclaimed.
|
|
624
|
+
"""
|
|
625
|
+
async with self._pool.acquire() as conn:
|
|
626
|
+
result = await conn.fetchval('SELECT reclaim_expired_task_runs()')
|
|
627
|
+
if result and result > 0:
|
|
628
|
+
logger.info(f'Reclaimed {result} expired task run leases')
|
|
629
|
+
return result or 0
|
|
630
|
+
|
|
631
|
+
def _row_to_task_run(self, row) -> TaskRun:
|
|
632
|
+
"""Convert database row to TaskRun object."""
|
|
633
|
+
import json as json_module
|
|
634
|
+
|
|
635
|
+
# Parse required_capabilities from JSON if present
|
|
636
|
+
required_capabilities = None
|
|
637
|
+
if row.get('required_capabilities'):
|
|
638
|
+
try:
|
|
639
|
+
caps = row['required_capabilities']
|
|
640
|
+
if isinstance(caps, str):
|
|
641
|
+
required_capabilities = json_module.loads(caps)
|
|
642
|
+
elif isinstance(caps, list):
|
|
643
|
+
required_capabilities = caps
|
|
644
|
+
except (json_module.JSONDecodeError, TypeError):
|
|
645
|
+
pass
|
|
646
|
+
|
|
647
|
+
return TaskRun(
|
|
648
|
+
id=row['id'],
|
|
649
|
+
task_id=row['task_id'],
|
|
650
|
+
user_id=row['user_id'],
|
|
651
|
+
template_id=row['template_id'],
|
|
652
|
+
automation_id=row['automation_id'],
|
|
653
|
+
status=TaskRunStatus(row['status']),
|
|
654
|
+
priority=row['priority'],
|
|
655
|
+
lease_owner=row['lease_owner'],
|
|
656
|
+
lease_expires_at=row['lease_expires_at'],
|
|
657
|
+
attempts=row['attempts'],
|
|
658
|
+
max_attempts=row['max_attempts'],
|
|
659
|
+
last_error=row['last_error'],
|
|
660
|
+
started_at=row['started_at'],
|
|
661
|
+
completed_at=row['completed_at'],
|
|
662
|
+
runtime_seconds=row['runtime_seconds'],
|
|
663
|
+
result_summary=row['result_summary'],
|
|
664
|
+
result_full=row['result_full'],
|
|
665
|
+
notify_email=row['notify_email'],
|
|
666
|
+
notify_webhook_url=row['notify_webhook_url'],
|
|
667
|
+
notification_sent=row['notification_sent'],
|
|
668
|
+
# Routing fields
|
|
669
|
+
target_agent_name=row.get('target_agent_name'),
|
|
670
|
+
required_capabilities=required_capabilities,
|
|
671
|
+
deadline_at=row.get('deadline_at'),
|
|
672
|
+
routing_failed_at=row.get('routing_failed_at'),
|
|
673
|
+
routing_failure_reason=row.get('routing_failure_reason'),
|
|
674
|
+
created_at=row['created_at'],
|
|
675
|
+
updated_at=row['updated_at'],
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
# Global task queue instance (initialized when DB pool is ready)
|
|
680
|
+
_task_queue: Optional[TaskQueue] = None
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def get_task_queue() -> Optional[TaskQueue]:
|
|
684
|
+
"""Get the global task queue instance."""
|
|
685
|
+
return _task_queue
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def set_task_queue(queue: TaskQueue) -> None:
|
|
689
|
+
"""Set the global task queue instance."""
|
|
690
|
+
global _task_queue
|
|
691
|
+
_task_queue = queue
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
async def enqueue_task(
|
|
695
|
+
task_id: str,
|
|
696
|
+
user_id: Optional[str] = None,
|
|
697
|
+
template_id: Optional[str] = None,
|
|
698
|
+
automation_id: Optional[str] = None,
|
|
699
|
+
priority: int = 0,
|
|
700
|
+
notify_email: Optional[str] = None,
|
|
701
|
+
notify_webhook_url: Optional[str] = None,
|
|
702
|
+
# Agent routing parameters
|
|
703
|
+
target_agent_name: Optional[str] = None,
|
|
704
|
+
required_capabilities: Optional[List[str]] = None,
|
|
705
|
+
deadline_at: Optional[datetime] = None,
|
|
706
|
+
) -> Optional[TaskRun]:
|
|
707
|
+
"""
|
|
708
|
+
Convenience function to enqueue a task.
|
|
709
|
+
|
|
710
|
+
Returns None if task queue is not initialized.
|
|
711
|
+
|
|
712
|
+
Args:
|
|
713
|
+
task_id: ID of the task to execute
|
|
714
|
+
user_id: Owner user ID (for concurrency limiting)
|
|
715
|
+
template_id: Template that generated this task (optional)
|
|
716
|
+
automation_id: Automation that generated this task (optional)
|
|
717
|
+
priority: Higher = more urgent (default 0)
|
|
718
|
+
notify_email: Email to notify on completion
|
|
719
|
+
notify_webhook_url: Webhook to call on completion
|
|
720
|
+
target_agent_name: If set, only this agent can claim the task
|
|
721
|
+
required_capabilities: List of capabilities the worker must have
|
|
722
|
+
deadline_at: If set, task fails if not claimed by this time
|
|
723
|
+
"""
|
|
724
|
+
queue = get_task_queue()
|
|
725
|
+
if queue is None:
|
|
726
|
+
logger.warning(
|
|
727
|
+
f'Task queue not initialized, cannot enqueue task {task_id}'
|
|
728
|
+
)
|
|
729
|
+
return None
|
|
730
|
+
|
|
731
|
+
return await queue.enqueue(
|
|
732
|
+
task_id=task_id,
|
|
733
|
+
user_id=user_id,
|
|
734
|
+
template_id=template_id,
|
|
735
|
+
automation_id=automation_id,
|
|
736
|
+
priority=priority,
|
|
737
|
+
notify_email=notify_email,
|
|
738
|
+
notify_webhook_url=notify_webhook_url,
|
|
739
|
+
target_agent_name=target_agent_name,
|
|
740
|
+
required_capabilities=required_capabilities,
|
|
741
|
+
deadline_at=deadline_at,
|
|
742
|
+
)
|