jettask 0.2.23__py3-none-any.whl → 0.2.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/__init__.py +2 -0
- jettask/cli.py +12 -8
- jettask/config/lua_scripts.py +37 -0
- jettask/config/nacos_config.py +1 -1
- jettask/core/app.py +313 -340
- jettask/core/container.py +4 -4
- jettask/{persistence → core}/namespace.py +93 -27
- jettask/core/task.py +16 -9
- jettask/core/unified_manager_base.py +136 -26
- jettask/db/__init__.py +67 -0
- jettask/db/base.py +137 -0
- jettask/{utils/db_connector.py → db/connector.py} +130 -26
- jettask/db/models/__init__.py +16 -0
- jettask/db/models/scheduled_task.py +196 -0
- jettask/db/models/task.py +77 -0
- jettask/db/models/task_run.py +85 -0
- jettask/executor/__init__.py +0 -15
- jettask/executor/core.py +76 -31
- jettask/executor/process_entry.py +29 -114
- jettask/executor/task_executor.py +4 -0
- jettask/messaging/event_pool.py +928 -685
- jettask/messaging/scanner.py +30 -0
- jettask/persistence/__init__.py +28 -103
- jettask/persistence/buffer.py +170 -0
- jettask/persistence/consumer.py +330 -249
- jettask/persistence/manager.py +304 -0
- jettask/persistence/persistence.py +391 -0
- jettask/scheduler/__init__.py +15 -3
- jettask/scheduler/{task_crud.py → database.py} +61 -57
- jettask/scheduler/loader.py +2 -2
- jettask/scheduler/{scheduler_coordinator.py → manager.py} +23 -6
- jettask/scheduler/models.py +14 -10
- jettask/scheduler/schedule.py +166 -0
- jettask/scheduler/scheduler.py +12 -11
- jettask/schemas/__init__.py +50 -1
- jettask/schemas/backlog.py +43 -6
- jettask/schemas/namespace.py +70 -19
- jettask/schemas/queue.py +19 -3
- jettask/schemas/responses.py +493 -0
- jettask/task/__init__.py +0 -2
- jettask/task/router.py +3 -0
- jettask/test_connection_monitor.py +1 -1
- jettask/utils/__init__.py +7 -5
- jettask/utils/db_init.py +8 -4
- jettask/utils/namespace_dep.py +167 -0
- jettask/utils/queue_matcher.py +186 -0
- jettask/utils/rate_limit/concurrency_limiter.py +7 -1
- jettask/utils/stream_backlog.py +1 -1
- jettask/webui/__init__.py +0 -1
- jettask/webui/api/__init__.py +4 -4
- jettask/webui/api/alerts.py +806 -71
- jettask/webui/api/example_refactored.py +400 -0
- jettask/webui/api/namespaces.py +390 -45
- jettask/webui/api/overview.py +300 -54
- jettask/webui/api/queues.py +971 -267
- jettask/webui/api/scheduled.py +1249 -56
- jettask/webui/api/settings.py +129 -7
- jettask/webui/api/workers.py +442 -0
- jettask/webui/app.py +46 -2329
- jettask/webui/middleware/__init__.py +6 -0
- jettask/webui/middleware/namespace_middleware.py +135 -0
- jettask/webui/services/__init__.py +146 -0
- jettask/webui/services/heartbeat_service.py +251 -0
- jettask/webui/services/overview_service.py +60 -51
- jettask/webui/services/queue_monitor_service.py +426 -0
- jettask/webui/services/redis_monitor_service.py +87 -0
- jettask/webui/services/settings_service.py +174 -111
- jettask/webui/services/task_monitor_service.py +222 -0
- jettask/webui/services/timeline_pg_service.py +452 -0
- jettask/webui/services/timeline_service.py +189 -0
- jettask/webui/services/worker_monitor_service.py +467 -0
- jettask/webui/utils/__init__.py +11 -0
- jettask/webui/utils/time_utils.py +122 -0
- jettask/worker/lifecycle.py +8 -2
- {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/METADATA +1 -1
- jettask-0.2.24.dist-info/RECORD +142 -0
- jettask/executor/executor.py +0 -338
- jettask/persistence/backlog_monitor.py +0 -567
- jettask/persistence/base.py +0 -2334
- jettask/persistence/db_manager.py +0 -516
- jettask/persistence/maintenance.py +0 -81
- jettask/persistence/message_consumer.py +0 -259
- jettask/persistence/models.py +0 -49
- jettask/persistence/offline_recovery.py +0 -196
- jettask/persistence/queue_discovery.py +0 -215
- jettask/persistence/task_persistence.py +0 -218
- jettask/persistence/task_updater.py +0 -583
- jettask/scheduler/add_execution_count.sql +0 -11
- jettask/scheduler/add_priority_field.sql +0 -26
- jettask/scheduler/add_scheduler_id.sql +0 -25
- jettask/scheduler/add_scheduler_id_index.sql +0 -10
- jettask/scheduler/make_scheduler_id_required.sql +0 -28
- jettask/scheduler/migrate_interval_seconds.sql +0 -9
- jettask/scheduler/performance_optimization.sql +0 -45
- jettask/scheduler/run_scheduler.py +0 -186
- jettask/scheduler/schema.sql +0 -84
- jettask/task/task_executor.py +0 -318
- jettask/webui/api/analytics.py +0 -323
- jettask/webui/config.py +0 -90
- jettask/webui/models/__init__.py +0 -3
- jettask/webui/models/namespace.py +0 -63
- jettask/webui/namespace_manager/__init__.py +0 -10
- jettask/webui/namespace_manager/multi.py +0 -593
- jettask/webui/namespace_manager/unified.py +0 -193
- jettask/webui/run.py +0 -46
- jettask-0.2.23.dist-info/RECORD +0 -145
- {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/WHEEL +0 -0
- {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/top_level.txt +0 -0
jettask/webui/api/queues.py
CHANGED
@@ -2,16 +2,17 @@
|
|
2
2
|
队列模块 - 队列管理、任务处理、队列统计和监控
|
3
3
|
提供轻量级的路由入口,业务逻辑在 QueueService 中实现
|
4
4
|
"""
|
5
|
-
from fastapi import APIRouter, HTTPException, Request, Query, Depends
|
6
|
-
from typing import Optional, Dict, Any
|
5
|
+
from fastapi import APIRouter, HTTPException, Request, Query, Path, Depends
|
6
|
+
from typing import Optional, Dict, Any, List
|
7
7
|
from datetime import datetime
|
8
8
|
import logging
|
9
9
|
|
10
10
|
from jettask.schemas import (
|
11
|
-
TimeRangeQuery,
|
11
|
+
TimeRangeQuery,
|
12
12
|
TrimQueueRequest,
|
13
13
|
TasksRequest,
|
14
14
|
TaskActionRequest,
|
15
|
+
BacklogLatestRequest,
|
15
16
|
BacklogTrendRequest
|
16
17
|
)
|
17
18
|
from jettask.webui.services.queue_service import QueueService
|
@@ -24,19 +25,63 @@ logger = logging.getLogger(__name__)
|
|
24
25
|
|
25
26
|
# ============ 队列基础管理 ============
|
26
27
|
|
27
|
-
@router.get(
|
28
|
-
|
28
|
+
@router.get(
|
29
|
+
"/{namespace}",
|
30
|
+
summary="获取命名空间队列列表",
|
31
|
+
description="获取指定命名空间下所有队列的基本信息和状态",
|
32
|
+
responses={
|
33
|
+
200: {
|
34
|
+
"description": "成功返回队列列表",
|
35
|
+
"content": {
|
36
|
+
"application/json": {
|
37
|
+
"example": {
|
38
|
+
"success": True,
|
39
|
+
"queues": [
|
40
|
+
{"name": "email_queue", "pending": 45, "processing": 3},
|
41
|
+
{"name": "sms_queue", "pending": 12, "processing": 1}
|
42
|
+
]
|
43
|
+
}
|
44
|
+
}
|
45
|
+
}
|
46
|
+
},
|
47
|
+
500: {"description": "服务器内部错误"}
|
48
|
+
}
|
49
|
+
)
|
50
|
+
async def get_queues(
|
51
|
+
request: Request,
|
52
|
+
namespace: str = Path(..., description="命名空间名称", example="default")
|
53
|
+
) -> Dict[str, Any]:
|
29
54
|
"""
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
55
|
+
## 获取命名空间队列列表
|
56
|
+
|
57
|
+
获取指定命名空间下所有队列的基本信息,包括队列名称和任务统计。
|
58
|
+
|
59
|
+
**返回信息包括**:
|
60
|
+
- 队列名称
|
61
|
+
- 待处理任务数
|
62
|
+
- 处理中任务数
|
63
|
+
- 其他队列状态信息
|
64
|
+
|
65
|
+
**使用场景**:
|
66
|
+
- 队列管理页面列表
|
67
|
+
- 队列状态监控
|
68
|
+
- 队列选择器
|
69
|
+
|
70
|
+
**示例请求**:
|
71
|
+
```bash
|
72
|
+
curl -X GET "http://localhost:8001/api/v1/queues/default"
|
73
|
+
```
|
74
|
+
|
75
|
+
**注意事项**:
|
76
|
+
- 返回该命名空间下的所有队列
|
77
|
+
- 数据实时从 Redis 获取
|
78
|
+
- 包含队列的基本统计信息
|
34
79
|
"""
|
35
80
|
try:
|
36
81
|
app = request.app
|
37
82
|
if not app or not hasattr(app.state, 'namespace_data_access'):
|
38
83
|
raise HTTPException(status_code=500, detail="Namespace data access not initialized")
|
39
|
-
|
84
|
+
|
40
85
|
namespace_data_access = app.state.namespace_data_access
|
41
86
|
return await QueueService.get_queues_by_namespace(namespace_data_access, namespace)
|
42
87
|
except Exception as e:
|
@@ -44,122 +89,67 @@ async def get_queues(request: Request, namespace: str):
|
|
44
89
|
raise HTTPException(status_code=500, detail=str(e))
|
45
90
|
|
46
91
|
|
47
|
-
@router.get("/detail")
|
48
|
-
async def get_queues_detail(request: Request):
|
49
|
-
"""获取队列详细信息"""
|
50
|
-
try:
|
51
|
-
app = request.app
|
52
|
-
if not app or not hasattr(app.state, 'data_access'):
|
53
|
-
raise HTTPException(status_code=500, detail="Data access not initialized")
|
54
|
-
|
55
|
-
data_access = app.state.data_access
|
56
|
-
return await QueueService.get_queues_detail(data_access)
|
57
|
-
except Exception as e:
|
58
|
-
logger.error(f"获取队列详细信息失败: {e}")
|
59
|
-
raise HTTPException(status_code=500, detail=str(e))
|
60
92
|
|
61
93
|
|
62
|
-
@router.
|
63
|
-
|
64
|
-
""
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
"""
|
80
|
-
裁剪队列到指定长度
|
81
|
-
|
82
|
-
Args:
|
83
|
-
queue_name: 队列名称
|
84
|
-
request: 裁剪请求参数
|
94
|
+
@router.get(
|
95
|
+
"/stats-v2/{namespace}",
|
96
|
+
summary="获取队列统计信息 v2",
|
97
|
+
description="获取队列的详细统计信息,支持消费者组和优先级队列",
|
98
|
+
responses={
|
99
|
+
200: {"description": "成功返回队列统计"},
|
100
|
+
500: {"description": "服务器内部错误"}
|
101
|
+
}
|
102
|
+
)
|
103
|
+
async def get_queue_stats(
|
104
|
+
request: Request,
|
105
|
+
namespace: str = Path(..., description="命名空间名称", example="default"),
|
106
|
+
queue: Optional[str] = Query(None, description="队列名称,为空则返回所有队列", example="email_queue"),
|
107
|
+
start_time: Optional[datetime] = Query(None, description="开始时间(ISO格式)"),
|
108
|
+
end_time: Optional[datetime] = Query(None, description="结束时间(ISO格式)"),
|
109
|
+
time_range: Optional[str] = Query(None, description="时间范围(如 1h, 24h, 7d)", example="24h")
|
110
|
+
) -> Dict[str, Any]:
|
85
111
|
"""
|
86
|
-
|
87
|
-
return await QueueService.trim_queue(queue_name, request.max_length)
|
88
|
-
except Exception as e:
|
89
|
-
logger.error(f"裁剪队列失败: {e}")
|
90
|
-
raise HTTPException(status_code=500, detail=str(e))
|
112
|
+
## 获取队列统计信息 v2
|
91
113
|
|
114
|
+
获取队列的详细统计信息,支持消费者组详情和优先级队列统计。
|
92
115
|
|
93
|
-
|
116
|
+
**增强特性**:
|
117
|
+
- ✅ 支持消费者组详细统计
|
118
|
+
- ✅ 支持优先级队列分析
|
119
|
+
- ✅ 支持时间范围筛选
|
120
|
+
- ✅ 支持单队列或全部队列查询
|
94
121
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
Args:
|
101
|
-
request: FastAPI request对象,包含JSON body
|
102
|
-
"""
|
103
|
-
try:
|
104
|
-
app = request.app
|
105
|
-
if not app or not hasattr(app.state, 'data_access'):
|
106
|
-
raise HTTPException(status_code=500, detail="Data access not initialized")
|
107
|
-
|
108
|
-
# 解析请求体
|
109
|
-
body = await request.json()
|
110
|
-
|
111
|
-
# 创建TimeRangeQuery对象
|
112
|
-
from jettask.schemas import TimeRangeQuery
|
113
|
-
query = TimeRangeQuery(**body)
|
114
|
-
|
115
|
-
data_access = app.state.data_access
|
116
|
-
return await QueueService.get_queue_flow_rates(data_access, query)
|
117
|
-
except Exception as e:
|
118
|
-
logger.error(f"获取队列流量速率失败: {e}")
|
119
|
-
import traceback
|
120
|
-
traceback.print_exc()
|
121
|
-
raise HTTPException(status_code=500, detail=str(e))
|
122
|
+
**返回信息包括**:
|
123
|
+
- 队列基本统计
|
124
|
+
- 消费者组状态和积压
|
125
|
+
- 优先级队列分布
|
126
|
+
- 时间范围内的任务统计
|
122
127
|
|
128
|
+
**使用场景**:
|
129
|
+
- 队列详细监控
|
130
|
+
- 消费者组管理
|
131
|
+
- 优先级队列分析
|
132
|
+
- 性能优化
|
123
133
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
app = request.app
|
129
|
-
if not app or not hasattr(app.state, 'data_access'):
|
130
|
-
raise HTTPException(status_code=500, detail="Data access not initialized")
|
131
|
-
|
132
|
-
data_access = app.state.data_access
|
133
|
-
return await QueueService.get_global_stats(data_access)
|
134
|
-
except Exception as e:
|
135
|
-
logger.error(f"获取全局统计信息失败: {e}")
|
136
|
-
raise HTTPException(status_code=500, detail=str(e))
|
134
|
+
**示例请求**:
|
135
|
+
```bash
|
136
|
+
# 获取指定队列的24小时统计
|
137
|
+
curl -X GET "http://localhost:8001/api/v1/queues/stats-v2/default?queue=email_queue&time_range=24h"
|
137
138
|
|
139
|
+
# 获取所有队列统计
|
140
|
+
curl -X GET "http://localhost:8001/api/v1/queues/stats-v2/default?time_range=1h"
|
141
|
+
```
|
138
142
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
queue: Optional[str] = Query(None, description="队列名称"),
|
144
|
-
start_time: Optional[datetime] = None,
|
145
|
-
end_time: Optional[datetime] = None,
|
146
|
-
time_range: Optional[str] = None
|
147
|
-
):
|
148
|
-
"""
|
149
|
-
获取队列统计信息v2 - 支持消费者组详情和优先级队列
|
150
|
-
|
151
|
-
Args:
|
152
|
-
namespace: 命名空间
|
153
|
-
queue: 可选,筛选特定队列
|
154
|
-
start_time: 开始时间
|
155
|
-
end_time: 结束时间
|
156
|
-
time_range: 时间范围
|
143
|
+
**注意事项**:
|
144
|
+
- v2 版本提供更详细的统计信息
|
145
|
+
- 消费者组数据仅在使用 Stream 时可用
|
146
|
+
- 建议使用时间范围参数限制数据量
|
157
147
|
"""
|
158
148
|
try:
|
159
149
|
app = request.app
|
160
150
|
if not app or not hasattr(app.state, 'namespace_data_access'):
|
161
151
|
raise HTTPException(status_code=500, detail="Namespace data access not initialized")
|
162
|
-
|
152
|
+
|
163
153
|
namespace_data_access = app.state.namespace_data_access
|
164
154
|
return await QueueService.get_queue_stats_v2(
|
165
155
|
namespace_data_access, namespace, queue, start_time, end_time, time_range
|
@@ -171,20 +161,69 @@ async def get_queue_stats_v2(
|
|
171
161
|
|
172
162
|
# ============ 消费者组统计 ============
|
173
163
|
|
174
|
-
@router.get(
|
175
|
-
|
164
|
+
@router.get(
|
165
|
+
"/consumer-groups/{namespace}/{group_name}/stats",
|
166
|
+
summary="获取消费者组统计",
|
167
|
+
description="获取指定消费者组的详细统计信息和积压情况",
|
168
|
+
responses={
|
169
|
+
200: {
|
170
|
+
"description": "成功返回消费者组统计",
|
171
|
+
"content": {
|
172
|
+
"application/json": {
|
173
|
+
"example": {
|
174
|
+
"group_name": "email_workers",
|
175
|
+
"pending_messages": 120,
|
176
|
+
"consumers": 5,
|
177
|
+
"lag": 95,
|
178
|
+
"last_delivered_id": "1697644800000-0"
|
179
|
+
}
|
180
|
+
}
|
181
|
+
}
|
182
|
+
},
|
183
|
+
400: {"description": "请求参数错误"},
|
184
|
+
500: {"description": "服务器内部错误"}
|
185
|
+
}
|
186
|
+
)
|
187
|
+
async def get_consumer_group_stats(
|
188
|
+
request: Request,
|
189
|
+
namespace: str = Path(..., description="命名空间名称", example="default"),
|
190
|
+
group_name: str = Path(..., description="消费者组名称", example="email_workers")
|
191
|
+
) -> Dict[str, Any]:
|
176
192
|
"""
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
193
|
+
## 获取消费者组统计
|
194
|
+
|
195
|
+
获取 Redis Stream 消费者组的详细统计信息,包括积压、消费者数量等。
|
196
|
+
|
197
|
+
**返回信息包括**:
|
198
|
+
- 消费者组名称
|
199
|
+
- 待处理消息数 (pending)
|
200
|
+
- 活跃消费者数量
|
201
|
+
- 消费延迟 (lag)
|
202
|
+
- 最后交付的消息 ID
|
203
|
+
- 各消费者的详细状态
|
204
|
+
|
205
|
+
**使用场景**:
|
206
|
+
- 消费者组监控
|
207
|
+
- 积压问题诊断
|
208
|
+
- 消费者负载均衡
|
209
|
+
- 性能优化
|
210
|
+
|
211
|
+
**示例请求**:
|
212
|
+
```bash
|
213
|
+
curl -X GET "http://localhost:8001/api/v1/queues/consumer-groups/default/email_workers/stats"
|
214
|
+
```
|
215
|
+
|
216
|
+
**注意事项**:
|
217
|
+
- 仅适用于使用 Redis Stream 的队列
|
218
|
+
- 消费者组需提前创建
|
219
|
+
- lag 值表示该组落后的消息数量
|
220
|
+
- 建议定期监控 pending 和 lag 指标
|
182
221
|
"""
|
183
222
|
try:
|
184
223
|
app = request.app
|
185
224
|
if not app or not hasattr(app.state, 'namespace_data_access'):
|
186
225
|
raise HTTPException(status_code=500, detail="Namespace data access not initialized")
|
187
|
-
|
226
|
+
|
188
227
|
namespace_data_access = app.state.namespace_data_access
|
189
228
|
return await QueueService.get_consumer_group_stats(namespace_data_access, namespace, group_name)
|
190
229
|
except ValueError as e:
|
@@ -196,26 +235,75 @@ async def get_consumer_group_stats(request: Request, namespace: str, group_name:
|
|
196
235
|
|
197
236
|
# ============ Stream积压监控 ============
|
198
237
|
|
199
|
-
@router.get(
|
238
|
+
@router.get(
|
239
|
+
"/stream-backlog/{namespace}",
|
240
|
+
summary="获取 Stream 积压监控数据",
|
241
|
+
description="获取 Redis Stream 的积压监控数据和历史趋势",
|
242
|
+
responses={
|
243
|
+
200: {
|
244
|
+
"description": "成功返回 Stream 积压数据",
|
245
|
+
"content": {
|
246
|
+
"application/json": {
|
247
|
+
"example": {
|
248
|
+
"stream_name": "task_stream",
|
249
|
+
"current_length": 1500,
|
250
|
+
"consumer_groups": 3,
|
251
|
+
"total_pending": 250,
|
252
|
+
"history": [
|
253
|
+
{"timestamp": "2025-10-18T10:00:00Z", "length": 1400},
|
254
|
+
{"timestamp": "2025-10-18T11:00:00Z", "length": 1500}
|
255
|
+
]
|
256
|
+
}
|
257
|
+
}
|
258
|
+
}
|
259
|
+
},
|
260
|
+
500: {"description": "服务器内部错误"}
|
261
|
+
}
|
262
|
+
)
|
200
263
|
async def get_stream_backlog(
|
201
264
|
request: Request,
|
202
|
-
namespace: str,
|
203
|
-
stream_name: Optional[str] = Query(None, description="Stream
|
204
|
-
hours: int = Query(24, description="查询最近多少小时的数据")
|
205
|
-
):
|
265
|
+
namespace: str = Path(..., description="命名空间名称", example="default"),
|
266
|
+
stream_name: Optional[str] = Query(None, description="Stream 名称,为空则返回所有", example="task_stream"),
|
267
|
+
hours: int = Query(24, ge=1, le=168, description="查询最近多少小时的数据", example=24)
|
268
|
+
) -> Dict[str, Any]:
|
206
269
|
"""
|
207
|
-
获取Stream积压监控数据
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
270
|
+
## 获取 Stream 积压监控数据
|
271
|
+
|
272
|
+
获取 Redis Stream 的积压情况和历史趋势数据。
|
273
|
+
|
274
|
+
**返回信息包括**:
|
275
|
+
- Stream 当前长度
|
276
|
+
- 消费者组数量
|
277
|
+
- 总待处理消息数
|
278
|
+
- 历史趋势数据(按小时)
|
279
|
+
- 积压率变化趋势
|
280
|
+
|
281
|
+
**使用场景**:
|
282
|
+
- Stream 积压监控
|
283
|
+
- 容量规划
|
284
|
+
- 性能趋势分析
|
285
|
+
- 异常检测
|
286
|
+
|
287
|
+
**示例请求**:
|
288
|
+
```bash
|
289
|
+
# 获取指定 Stream 最近24小时积压数据
|
290
|
+
curl -X GET "http://localhost:8001/api/v1/queues/stream-backlog/default?stream_name=task_stream&hours=24"
|
291
|
+
|
292
|
+
# 获取所有 Stream 的积压数据
|
293
|
+
curl -X GET "http://localhost:8001/api/v1/queues/stream-backlog/default?hours=48"
|
294
|
+
```
|
295
|
+
|
296
|
+
**注意事项**:
|
297
|
+
- 仅适用于使用 Redis Stream 的队列
|
298
|
+
- hours 参数范围: 1-168 (7天)
|
299
|
+
- 历史数据按小时聚合
|
300
|
+
- 建议定期监控积压趋势
|
213
301
|
"""
|
214
302
|
try:
|
215
303
|
app = request.app
|
216
304
|
if not app or not hasattr(app.state, 'data_access'):
|
217
305
|
raise HTTPException(status_code=500, detail="Data access not initialized")
|
218
|
-
|
306
|
+
|
219
307
|
data_access = app.state.data_access
|
220
308
|
return await QueueService.get_stream_backlog(data_access, namespace, stream_name, hours)
|
221
309
|
except Exception as e:
|
@@ -223,19 +311,68 @@ async def get_stream_backlog(
|
|
223
311
|
raise HTTPException(status_code=500, detail=str(e))
|
224
312
|
|
225
313
|
|
226
|
-
@router.get(
|
227
|
-
|
314
|
+
@router.get(
|
315
|
+
"/stream-backlog/{namespace}/summary",
|
316
|
+
summary="获取 Stream 积压汇总",
|
317
|
+
description="获取命名空间下所有 Stream 的积压汇总信息",
|
318
|
+
responses={
|
319
|
+
200: {
|
320
|
+
"description": "成功返回积压汇总",
|
321
|
+
"content": {
|
322
|
+
"application/json": {
|
323
|
+
"example": {
|
324
|
+
"total_streams": 5,
|
325
|
+
"total_length": 7500,
|
326
|
+
"total_pending": 850,
|
327
|
+
"avg_backlog_rate": 12.5,
|
328
|
+
"streams": [
|
329
|
+
{"name": "task_stream", "length": 1500, "pending": 250},
|
330
|
+
{"name": "email_stream", "length": 2000, "pending": 180}
|
331
|
+
]
|
332
|
+
}
|
333
|
+
}
|
334
|
+
}
|
335
|
+
},
|
336
|
+
500: {"description": "服务器内部错误"}
|
337
|
+
}
|
338
|
+
)
|
339
|
+
async def get_stream_backlog_summary(
|
340
|
+
request: Request,
|
341
|
+
namespace: str = Path(..., description="命名空间名称", example="default")
|
342
|
+
) -> Dict[str, Any]:
|
228
343
|
"""
|
229
|
-
获取Stream
|
230
|
-
|
231
|
-
|
232
|
-
|
344
|
+
## 获取 Stream 积压汇总
|
345
|
+
|
346
|
+
获取指定命名空间下所有 Redis Stream 的积压汇总统计。
|
347
|
+
|
348
|
+
**返回信息包括**:
|
349
|
+
- Stream 总数
|
350
|
+
- 总消息数
|
351
|
+
- 总待处理消息数
|
352
|
+
- 平均积压率
|
353
|
+
- 各 Stream 的简要信息
|
354
|
+
|
355
|
+
**使用场景**:
|
356
|
+
- 全局积压监控
|
357
|
+
- 命名空间健康检查
|
358
|
+
- 快速问题定位
|
359
|
+
- 管理报表
|
360
|
+
|
361
|
+
**示例请求**:
|
362
|
+
```bash
|
363
|
+
curl -X GET "http://localhost:8001/api/v1/queues/stream-backlog/default/summary"
|
364
|
+
```
|
365
|
+
|
366
|
+
**注意事项**:
|
367
|
+
- 实时计算,可能有延迟
|
368
|
+
- 仅包含使用 Stream 的队列
|
369
|
+
- 建议配合详细接口使用
|
233
370
|
"""
|
234
371
|
try:
|
235
372
|
app = request.app
|
236
373
|
if not app or not hasattr(app.state, 'data_access'):
|
237
374
|
raise HTTPException(status_code=500, detail="Data access not initialized")
|
238
|
-
|
375
|
+
|
239
376
|
data_access = app.state.data_access
|
240
377
|
return await QueueService.get_stream_backlog_summary(data_access, namespace)
|
241
378
|
except Exception as e:
|
@@ -245,70 +382,252 @@ async def get_stream_backlog_summary(request: Request, namespace: str):
|
|
245
382
|
|
246
383
|
# ============ 队列积压监控 ============
|
247
384
|
|
248
|
-
@router.post(
|
249
|
-
|
385
|
+
@router.post(
|
386
|
+
"/backlog/latest/{namespace}",
|
387
|
+
summary="获取最新积压数据快照",
|
388
|
+
description="获取指定命名空间下队列的最新积压数据快照",
|
389
|
+
responses={
|
390
|
+
200: {
|
391
|
+
"description": "成功返回积压快照数据",
|
392
|
+
"content": {
|
393
|
+
"application/json": {
|
394
|
+
"example": {
|
395
|
+
"success": True,
|
396
|
+
"namespace": "default",
|
397
|
+
"timestamp": "2025-10-18T10:30:00Z",
|
398
|
+
"snapshots": [
|
399
|
+
{
|
400
|
+
"queue_name": "email_queue",
|
401
|
+
"pending_count": 120,
|
402
|
+
"processing_count": 8,
|
403
|
+
"completed_count": 5430,
|
404
|
+
"failed_count": 12,
|
405
|
+
"queue_size": 128,
|
406
|
+
"oldest_task_age": 45
|
407
|
+
}
|
408
|
+
]
|
409
|
+
}
|
410
|
+
}
|
411
|
+
}
|
412
|
+
},
|
413
|
+
500: {"description": "服务器内部错误"}
|
414
|
+
}
|
415
|
+
)
|
416
|
+
async def get_latest_backlog(
|
417
|
+
request: Request,
|
418
|
+
namespace: str = Path(..., description="命名空间名称", example="default"),
|
419
|
+
backlog_request: BacklogLatestRequest = ...
|
420
|
+
) -> Dict[str, Any]:
|
250
421
|
"""
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
422
|
+
## 获取最新积压数据快照
|
423
|
+
|
424
|
+
获取指定命名空间下一个或多个队列的最新积压数据快照,用于实时监控队列状态。
|
425
|
+
|
426
|
+
**返回信息包括**:
|
427
|
+
- 待处理任务数 (pending_count)
|
428
|
+
- 处理中任务数 (processing_count)
|
429
|
+
- 已完成任务数 (completed_count)
|
430
|
+
- 失败任务数 (failed_count)
|
431
|
+
- 队列大小 (queue_size)
|
432
|
+
- 最老任务年龄(秒)
|
433
|
+
|
434
|
+
**使用场景**:
|
435
|
+
- 实时积压监控
|
436
|
+
- 队列健康检查
|
437
|
+
- 告警触发依据
|
438
|
+
- 运维大盘展示
|
439
|
+
|
440
|
+
**示例请求**:
|
441
|
+
```bash
|
442
|
+
# 获取指定队列的积压快照
|
443
|
+
curl -X POST "http://localhost:8001/api/v1/queues/backlog/latest/default" \\
|
444
|
+
-H "Content-Type: application/json" \\
|
445
|
+
-d '{
|
446
|
+
"queues": ["email_queue", "sms_queue"]
|
447
|
+
}'
|
448
|
+
|
449
|
+
# 获取所有队列的积压快照
|
450
|
+
curl -X POST "http://localhost:8001/api/v1/queues/backlog/latest/default" \\
|
451
|
+
-H "Content-Type: application/json" \\
|
452
|
+
-d '{
|
453
|
+
"queues": []
|
454
|
+
}'
|
455
|
+
```
|
456
|
+
|
457
|
+
**注意事项**:
|
458
|
+
- 数据为实时快照,反映当前时刻的队列状态
|
459
|
+
- queues 参数为空或不提供时,返回所有队列的数据
|
460
|
+
- oldest_task_age 为空表示队列中没有待处理任务
|
461
|
+
- 建议配合告警规则使用,及时发现积压问题
|
255
462
|
"""
|
256
463
|
try:
|
257
464
|
app = request.app
|
258
465
|
if not app or not hasattr(app.state, 'data_access'):
|
259
466
|
raise HTTPException(status_code=500, detail="Data access not initialized")
|
260
|
-
|
467
|
+
|
261
468
|
data_access = app.state.data_access
|
262
|
-
|
263
|
-
|
264
|
-
|
469
|
+
queues = backlog_request.queues or []
|
470
|
+
|
265
471
|
# TODO: 调用QueueService的积压监控方法
|
472
|
+
# snapshots = await QueueService.get_latest_backlog(data_access, namespace, queues)
|
473
|
+
|
266
474
|
return {
|
267
475
|
"success": True,
|
268
476
|
"namespace": namespace,
|
269
|
-
"queues": queues,
|
270
|
-
"data": [],
|
271
477
|
"timestamp": datetime.now().isoformat(),
|
272
|
-
"
|
478
|
+
"snapshots": [],
|
479
|
+
"message": "Backlog monitoring endpoint - implementation pending"
|
273
480
|
}
|
274
481
|
except Exception as e:
|
275
482
|
logger.error(f"获取最新积压数据失败: {e}")
|
276
483
|
raise HTTPException(status_code=500, detail=str(e))
|
277
484
|
|
278
485
|
|
279
|
-
@router.post(
|
280
|
-
|
486
|
+
@router.post(
|
487
|
+
"/backlog/trend/{namespace}",
|
488
|
+
summary="获取队列积压趋势",
|
489
|
+
description="获取指定队列在一段时间内的积压趋势数据,支持多种时间粒度",
|
490
|
+
responses={
|
491
|
+
200: {
|
492
|
+
"description": "成功返回积压趋势数据",
|
493
|
+
"content": {
|
494
|
+
"application/json": {
|
495
|
+
"example": {
|
496
|
+
"success": True,
|
497
|
+
"namespace": "default",
|
498
|
+
"queue_name": "email_queue",
|
499
|
+
"time_range": "1h",
|
500
|
+
"interval": "5m",
|
501
|
+
"timestamps": [
|
502
|
+
"2025-10-18T10:00:00Z",
|
503
|
+
"2025-10-18T10:05:00Z",
|
504
|
+
"2025-10-18T10:10:00Z"
|
505
|
+
],
|
506
|
+
"metrics": {
|
507
|
+
"pending": [120, 115, 108],
|
508
|
+
"processing": [8, 10, 9],
|
509
|
+
"completed": [5430, 5445, 5462],
|
510
|
+
"failed": [12, 12, 13]
|
511
|
+
},
|
512
|
+
"statistics": {
|
513
|
+
"peak_pending": 120,
|
514
|
+
"avg_pending": 114.3,
|
515
|
+
"avg_throughput": 10.7,
|
516
|
+
"overall_success_rate": 99.76
|
517
|
+
}
|
518
|
+
}
|
519
|
+
}
|
520
|
+
}
|
521
|
+
},
|
522
|
+
400: {"description": "请求参数错误"},
|
523
|
+
500: {"description": "服务器内部错误"}
|
524
|
+
}
|
525
|
+
)
|
526
|
+
async def get_backlog_trend(
|
527
|
+
request: Request,
|
528
|
+
namespace: str = Path(..., description="命名空间名称", example="default"),
|
529
|
+
trend_request: BacklogTrendRequest = ...
|
530
|
+
) -> Dict[str, Any]:
|
281
531
|
"""
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
532
|
+
## 获取队列积压趋势
|
533
|
+
|
534
|
+
获取指定队列在一段时间内的积压趋势数据,包括各项指标的时间序列和统计摘要。
|
535
|
+
|
536
|
+
**支持的时间范围**:
|
537
|
+
- `1h`: 最近1小时
|
538
|
+
- `6h`: 最近6小时
|
539
|
+
- `1d` 或 `24h`: 最近1天
|
540
|
+
- `7d`: 最近7天
|
541
|
+
- 或使用 `start_time` 和 `end_time` 指定精确时间范围
|
542
|
+
|
543
|
+
**支持的时间间隔**:
|
544
|
+
- `1m`: 1分钟粒度
|
545
|
+
- `5m`: 5分钟粒度
|
546
|
+
- `15m`: 15分钟粒度
|
547
|
+
- `1h`: 1小时粒度
|
548
|
+
|
549
|
+
**支持的指标类型**:
|
550
|
+
- `pending`: 待处理任务数
|
551
|
+
- `processing`: 处理中任务数
|
552
|
+
- `completed`: 已完成任务数
|
553
|
+
- `failed`: 失败任务数
|
554
|
+
|
555
|
+
**返回信息包括**:
|
556
|
+
- 时间戳序列
|
557
|
+
- 各指标的时间序列数据
|
558
|
+
- 统计摘要(峰值、平均值、成功率等)
|
559
|
+
|
560
|
+
**使用场景**:
|
561
|
+
- 积压趋势分析
|
562
|
+
- 性能问题诊断
|
563
|
+
- 容量规划
|
564
|
+
- 历史数据回溯
|
565
|
+
- 运维报表生成
|
566
|
+
|
567
|
+
**示例请求**:
|
568
|
+
```bash
|
569
|
+
# 获取指定队列最近1小时的积压趋势
|
570
|
+
curl -X POST "http://localhost:8001/api/v1/queues/backlog/trend/default" \\
|
571
|
+
-H "Content-Type: application/json" \\
|
572
|
+
-d '{
|
573
|
+
"queue_name": "email_queue",
|
574
|
+
"time_range": "1h",
|
575
|
+
"interval": "5m",
|
576
|
+
"metrics": ["pending", "processing"]
|
577
|
+
}'
|
578
|
+
|
579
|
+
# 使用精确时间范围查询
|
580
|
+
curl -X POST "http://localhost:8001/api/v1/queues/backlog/trend/default" \\
|
581
|
+
-H "Content-Type: application/json" \\
|
582
|
+
-d '{
|
583
|
+
"queue_name": "sms_queue",
|
584
|
+
"start_time": "2025-10-18T09:00:00Z",
|
585
|
+
"end_time": "2025-10-18T10:00:00Z",
|
586
|
+
"interval": "1m",
|
587
|
+
"metrics": ["pending", "processing", "completed", "failed"]
|
588
|
+
}'
|
589
|
+
|
590
|
+
# 获取所有队列的趋势(不指定queue_name)
|
591
|
+
curl -X POST "http://localhost:8001/api/v1/queues/backlog/trend/default" \\
|
592
|
+
-H "Content-Type: application/json" \\
|
593
|
+
-d '{
|
594
|
+
"time_range": "24h",
|
595
|
+
"interval": "1h"
|
596
|
+
}'
|
597
|
+
```
|
598
|
+
|
599
|
+
**注意事项**:
|
600
|
+
- `time_range` 和 `start_time/end_time` 二选一,优先使用 `start_time/end_time`
|
601
|
+
- 时间间隔越小,返回的数据点越多,建议根据时间范围选择合适的间隔
|
602
|
+
- 建议时间范围: 1h→5m, 6h→15m, 24h→1h, 7d→1h
|
603
|
+
- 统计摘要仅在请求所有指标时才完整
|
604
|
+
- 数据来源于持久化存储,可能有轻微延迟
|
286
605
|
"""
|
287
606
|
try:
|
288
607
|
app = request.app
|
289
608
|
if not app or not hasattr(app.state, 'data_access'):
|
290
609
|
raise HTTPException(status_code=500, detail="Data access not initialized")
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
# 从请求体中获取参数,namespace从路径参数获取
|
296
|
-
time_range = body.get('time_range', '1h')
|
297
|
-
queue_name = body.get('queue_name')
|
298
|
-
interval = body.get('interval', '5m')
|
299
|
-
metrics = body.get('metrics', ["pending", "processing", "completed", "failed"])
|
300
|
-
|
610
|
+
|
611
|
+
data_access = app.state.data_access
|
612
|
+
|
301
613
|
# TODO: 调用QueueService的积压趋势方法
|
614
|
+
# trend_data = await QueueService.get_backlog_trend(
|
615
|
+
# data_access, namespace, trend_request
|
616
|
+
# )
|
617
|
+
|
302
618
|
return {
|
303
619
|
"success": True,
|
304
620
|
"namespace": namespace,
|
305
|
-
"
|
306
|
-
"
|
307
|
-
"interval": interval,
|
308
|
-
"
|
621
|
+
"queue_name": trend_request.queue_name,
|
622
|
+
"time_range": trend_request.time_range,
|
623
|
+
"interval": trend_request.interval,
|
624
|
+
"timestamps": [],
|
625
|
+
"metrics": {},
|
309
626
|
"statistics": {},
|
310
|
-
"
|
627
|
+
"message": "Backlog trend endpoint - implementation pending"
|
311
628
|
}
|
629
|
+
except ValueError as e:
|
630
|
+
raise HTTPException(status_code=400, detail=str(e))
|
312
631
|
except Exception as e:
|
313
632
|
logger.error(f"获取积压趋势失败: {e}")
|
314
633
|
raise HTTPException(status_code=500, detail=str(e))
|
@@ -323,93 +642,106 @@ def get_task_service(request: Request) -> TaskService:
|
|
323
642
|
return TaskService(request.app.state.data_access)
|
324
643
|
|
325
644
|
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
645
|
+
|
646
|
+
@router.post(
|
647
|
+
"/tasks-v2/{namespace}",
|
648
|
+
summary="获取任务列表 v2",
|
649
|
+
description="获取任务列表v2版本,支持tasks和task_runs表连表查询,提供更丰富的查询和过滤功能",
|
650
|
+
responses={
|
651
|
+
200: {
|
652
|
+
"description": "成功返回任务列表",
|
653
|
+
"content": {
|
654
|
+
"application/json": {
|
655
|
+
"example": {
|
656
|
+
"success": True,
|
657
|
+
"data": [
|
658
|
+
{
|
659
|
+
"task_id": "task-001",
|
660
|
+
"queue_name": "email_queue",
|
661
|
+
"status": "completed",
|
662
|
+
"created_at": "2025-10-18T10:00:00Z",
|
663
|
+
"runs": []
|
664
|
+
}
|
665
|
+
],
|
666
|
+
"total": 150,
|
667
|
+
"page": 1,
|
668
|
+
"page_size": 20
|
669
|
+
}
|
670
|
+
}
|
671
|
+
}
|
672
|
+
},
|
673
|
+
400: {"description": "请求参数错误"},
|
674
|
+
500: {"description": "服务器内部错误"}
|
675
|
+
}
|
676
|
+
)
|
677
|
+
async def get_tasks_v2(request: Request, namespace: str):
|
335
678
|
"""
|
336
|
-
|
337
|
-
return await service.get_tasks_with_filters(
|
338
|
-
queue_name=request.queue_name,
|
339
|
-
page=request.page,
|
340
|
-
page_size=request.page_size,
|
341
|
-
filters=request.filters,
|
342
|
-
time_range=request.time_range,
|
343
|
-
start_time=request.start_time,
|
344
|
-
end_time=request.end_time
|
345
|
-
)
|
346
|
-
except ValueError as e:
|
347
|
-
raise HTTPException(status_code=400, detail=str(e))
|
348
|
-
except Exception as e:
|
349
|
-
raise HTTPException(status_code=500, detail=str(e))
|
679
|
+
## 获取任务列表 v2
|
350
680
|
|
681
|
+
获取任务列表的增强版本,支持 tasks 和 task_runs 表的连表查询,提供更强大的查询和过滤功能。
|
351
682
|
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
status: Optional[str] = None,
|
358
|
-
task_id: Optional[str] = None,
|
359
|
-
worker_id: Optional[str] = None,
|
360
|
-
service: TaskService = Depends(get_task_service)
|
361
|
-
) -> Dict[str, Any]:
|
362
|
-
"""
|
363
|
-
获取队列的任务列表(向后兼容旧版本)
|
364
|
-
"""
|
365
|
-
# 构建筛选条件
|
366
|
-
filters = []
|
367
|
-
if status:
|
368
|
-
filters.append({'field': 'status', 'operator': 'eq', 'value': status})
|
369
|
-
if task_id:
|
370
|
-
filters.append({'field': 'id', 'operator': 'eq', 'value': task_id})
|
371
|
-
if worker_id:
|
372
|
-
filters.append({'field': 'worker_id', 'operator': 'eq', 'value': worker_id})
|
373
|
-
|
374
|
-
try:
|
375
|
-
return await service.get_tasks_with_filters(
|
376
|
-
queue_name=queue_name,
|
377
|
-
page=page,
|
378
|
-
page_size=page_size,
|
379
|
-
filters=filters
|
380
|
-
)
|
381
|
-
except Exception as e:
|
382
|
-
raise HTTPException(status_code=500, detail=str(e))
|
683
|
+
**增强特性**:
|
684
|
+
- ✅ 支持多表连接查询
|
685
|
+
- ✅ 支持复杂过滤条件
|
686
|
+
- ✅ 支持排序和分页
|
687
|
+
- ✅ 返回任务执行历史
|
383
688
|
|
689
|
+
**请求体参数**:
|
690
|
+
```json
|
691
|
+
{
|
692
|
+
"queue_name": "email_queue", // 可选:队列名称
|
693
|
+
"status": "completed", // 可选:任务状态
|
694
|
+
"page": 1, // 可选:页码,默认1
|
695
|
+
"page_size": 20, // 可选:每页数量,默认20
|
696
|
+
"start_time": "2025-10-18T00:00:00Z", // 可选:开始时间
|
697
|
+
"end_time": "2025-10-18T23:59:59Z", // 可选:结束时间
|
698
|
+
"sort_by": "created_at", // 可选:排序字段
|
699
|
+
"sort_order": "desc" // 可选:排序方向 (asc/desc)
|
700
|
+
}
|
701
|
+
```
|
384
702
|
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
"""
|
392
|
-
获取单个任务的详细数据
|
393
|
-
|
394
|
-
包括task_data和result
|
395
|
-
"""
|
396
|
-
try:
|
397
|
-
task_details = await service.get_task_details(task_id, consumer_group)
|
398
|
-
return {
|
399
|
-
"success": True,
|
400
|
-
"data": task_details
|
401
|
-
}
|
402
|
-
except ValueError as e:
|
403
|
-
raise HTTPException(status_code=404, detail=str(e))
|
404
|
-
except Exception as e:
|
405
|
-
raise HTTPException(status_code=500, detail=str(e))
|
703
|
+
**支持的任务状态**:
|
704
|
+
- `pending`: 待处理
|
705
|
+
- `processing`: 处理中
|
706
|
+
- `completed`: 已完成
|
707
|
+
- `failed`: 失败
|
708
|
+
- `retrying`: 重试中
|
406
709
|
|
710
|
+
**使用场景**:
|
711
|
+
- 任务管理页面
|
712
|
+
- 任务历史查询
|
713
|
+
- 任务状态监控
|
714
|
+
- 故障排查
|
715
|
+
|
716
|
+
**示例请求**:
|
717
|
+
```bash
|
718
|
+
# 查询指定队列的已完成任务
|
719
|
+
curl -X POST "http://localhost:8001/api/v1/queues/tasks-v2/default" \\
|
720
|
+
-H "Content-Type: application/json" \\
|
721
|
+
-d '{
|
722
|
+
"queue_name": "email_queue",
|
723
|
+
"status": "completed",
|
724
|
+
"page": 1,
|
725
|
+
"page_size": 20
|
726
|
+
}'
|
727
|
+
|
728
|
+
# 查询指定时间范围的任务
|
729
|
+
curl -X POST "http://localhost:8001/api/v1/queues/tasks-v2/default" \\
|
730
|
+
-H "Content-Type: application/json" \\
|
731
|
+
-d '{
|
732
|
+
"start_time": "2025-10-18T00:00:00Z",
|
733
|
+
"end_time": "2025-10-18T23:59:59Z",
|
734
|
+
"sort_by": "created_at",
|
735
|
+
"sort_order": "desc"
|
736
|
+
}'
|
737
|
+
```
|
738
|
+
|
739
|
+
**注意事项**:
|
740
|
+
- v2 版本提供更详细的任务信息
|
741
|
+
- 包含任务的执行历史(runs)
|
742
|
+
- 建议使用分页避免一次查询过多数据
|
743
|
+
- 时间参数使用 ISO 8601 格式
|
407
744
|
|
408
|
-
@router.post("/tasks-v2/{namespace}")
|
409
|
-
async def get_tasks_v2(request: Request, namespace: str):
|
410
|
-
"""
|
411
|
-
获取任务列表v2 - 支持tasks和task_runs表连表查询
|
412
|
-
|
413
745
|
Args:
|
414
746
|
namespace: 命名空间
|
415
747
|
"""
|
@@ -433,11 +765,76 @@ async def get_tasks_v2(request: Request, namespace: str):
|
|
433
765
|
|
434
766
|
# ============ Redis监控 ============
|
435
767
|
|
436
|
-
@router.get(
|
768
|
+
@router.get(
|
769
|
+
"/redis/monitor/{namespace}",
|
770
|
+
summary="获取 Redis 性能监控数据",
|
771
|
+
description="获取指定命名空间的 Redis 实时性能监控数据,包括内存使用、连接数、命令统计等",
|
772
|
+
responses={
|
773
|
+
200: {
|
774
|
+
"description": "成功返回 Redis 监控数据",
|
775
|
+
"content": {
|
776
|
+
"application/json": {
|
777
|
+
"example": {
|
778
|
+
"success": True,
|
779
|
+
"namespace": "default",
|
780
|
+
"redis_info": {
|
781
|
+
"used_memory": "10485760",
|
782
|
+
"used_memory_human": "10M",
|
783
|
+
"connected_clients": "25",
|
784
|
+
"total_commands_processed": "1500000",
|
785
|
+
"instantaneous_ops_per_sec": "150",
|
786
|
+
"keyspace_hits": "98500",
|
787
|
+
"keyspace_misses": "1500"
|
788
|
+
},
|
789
|
+
"performance": {
|
790
|
+
"hit_rate": "98.5%",
|
791
|
+
"qps": 150,
|
792
|
+
"memory_fragmentation_ratio": 1.2
|
793
|
+
}
|
794
|
+
}
|
795
|
+
}
|
796
|
+
}
|
797
|
+
},
|
798
|
+
500: {"description": "服务器内部错误"}
|
799
|
+
}
|
800
|
+
)
|
437
801
|
async def get_redis_monitor(request: Request, namespace: str):
|
438
802
|
"""
|
439
|
-
获取Redis性能监控数据
|
440
|
-
|
803
|
+
## 获取 Redis 性能监控数据
|
804
|
+
|
805
|
+
获取指定命名空间的 Redis 实例的实时性能监控数据。
|
806
|
+
|
807
|
+
**监控指标包括**:
|
808
|
+
- **内存使用**: 已用内存、峰值内存、内存碎片率
|
809
|
+
- **连接信息**: 当前连接数、阻塞的客户端数
|
810
|
+
- **命令统计**: 总命令数、每秒操作数(QPS)
|
811
|
+
- **缓存命中**: 命中次数、未命中次数、命中率
|
812
|
+
- **持久化**: RDB/AOF 状态、最后保存时间
|
813
|
+
- **键空间**: 各数据库的键数量和过期键数量
|
814
|
+
|
815
|
+
**性能指标**:
|
816
|
+
- `hit_rate`: 缓存命中率
|
817
|
+
- `qps`: 每秒查询数
|
818
|
+
- `memory_fragmentation_ratio`: 内存碎片率
|
819
|
+
|
820
|
+
**使用场景**:
|
821
|
+
- Redis 性能监控
|
822
|
+
- 容量规划
|
823
|
+
- 性能优化
|
824
|
+
- 故障诊断
|
825
|
+
- 运维大盘
|
826
|
+
|
827
|
+
**示例请求**:
|
828
|
+
```bash
|
829
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/monitor/default"
|
830
|
+
```
|
831
|
+
|
832
|
+
**注意事项**:
|
833
|
+
- 数据实时获取,反映当前时刻的 Redis 状态
|
834
|
+
- 频繁调用可能对 Redis 性能有轻微影响
|
835
|
+
- 建议监控间隔设置为 5-10 秒
|
836
|
+
- 内存碎片率 > 1.5 时建议重启 Redis
|
837
|
+
|
441
838
|
Args:
|
442
839
|
namespace: 命名空间
|
443
840
|
"""
|
@@ -454,15 +851,74 @@ async def get_redis_monitor(request: Request, namespace: str):
|
|
454
851
|
raise HTTPException(status_code=500, detail=str(e))
|
455
852
|
|
456
853
|
|
457
|
-
@router.get(
|
854
|
+
@router.get(
|
855
|
+
"/redis/slow-log/{namespace}",
|
856
|
+
summary="获取 Redis 慢查询日志",
|
857
|
+
description="获取 Redis 的慢查询日志,用于诊断性能问题",
|
858
|
+
responses={
|
859
|
+
200: {
|
860
|
+
"description": "成功返回慢查询日志",
|
861
|
+
"content": {
|
862
|
+
"application/json": {
|
863
|
+
"example": {
|
864
|
+
"success": True,
|
865
|
+
"namespace": "default",
|
866
|
+
"slow_logs": [
|
867
|
+
{
|
868
|
+
"id": 12345,
|
869
|
+
"timestamp": 1697644800,
|
870
|
+
"duration_us": 15000,
|
871
|
+
"command": "KEYS pattern*",
|
872
|
+
"client_addr": "127.0.0.1:54321"
|
873
|
+
}
|
874
|
+
],
|
875
|
+
"total": 10
|
876
|
+
}
|
877
|
+
}
|
878
|
+
}
|
879
|
+
},
|
880
|
+
500: {"description": "服务器内部错误"}
|
881
|
+
}
|
882
|
+
)
|
458
883
|
async def get_redis_slow_log(
|
459
884
|
request: Request,
|
460
885
|
namespace: str,
|
461
|
-
limit: int = Query(10, description="
|
886
|
+
limit: int = Query(10, ge=1, le=100, description="返回记录数,范围 1-100", example=10)
|
462
887
|
):
|
463
888
|
"""
|
464
|
-
获取Redis慢查询日志
|
465
|
-
|
889
|
+
## 获取 Redis 慢查询日志
|
890
|
+
|
891
|
+
获取 Redis 实例的慢查询日志,帮助识别性能瓶颈和优化查询。
|
892
|
+
|
893
|
+
**日志信息包括**:
|
894
|
+
- 日志 ID
|
895
|
+
- 执行时间戳
|
896
|
+
- 执行耗时(微秒)
|
897
|
+
- 执行的命令
|
898
|
+
- 客户端地址和端口
|
899
|
+
|
900
|
+
**使用场景**:
|
901
|
+
- 性能问题诊断
|
902
|
+
- 慢查询优化
|
903
|
+
- Redis 性能分析
|
904
|
+
- 识别不合理的命令使用
|
905
|
+
|
906
|
+
**示例请求**:
|
907
|
+
```bash
|
908
|
+
# 获取最近10条慢查询日志
|
909
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/slow-log/default?limit=10"
|
910
|
+
|
911
|
+
# 获取最近50条慢查询日志
|
912
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/slow-log/default?limit=50"
|
913
|
+
```
|
914
|
+
|
915
|
+
**注意事项**:
|
916
|
+
- 慢查询阈值由 Redis 配置 `slowlog-log-slower-than` 决定(默认10ms)
|
917
|
+
- 日志按时间倒序返回(最新的在前)
|
918
|
+
- 慢查询日志存储在内存中,重启后清空
|
919
|
+
- 建议定期检查并优化慢查询
|
920
|
+
- 常见慢命令: KEYS、SMEMBERS、HGETALL(大集合)
|
921
|
+
|
466
922
|
Args:
|
467
923
|
namespace: 命名空间
|
468
924
|
limit: 返回记录数
|
@@ -480,11 +936,85 @@ async def get_redis_slow_log(
|
|
480
936
|
raise HTTPException(status_code=500, detail=str(e))
|
481
937
|
|
482
938
|
|
483
|
-
@router.get(
|
939
|
+
@router.get(
|
940
|
+
"/redis/command-stats/{namespace}",
|
941
|
+
summary="获取 Redis 命令统计",
|
942
|
+
description="获取 Redis 各类命令的执行统计信息,包括调用次数、耗时等",
|
943
|
+
responses={
|
944
|
+
200: {
|
945
|
+
"description": "成功返回命令统计",
|
946
|
+
"content": {
|
947
|
+
"application/json": {
|
948
|
+
"example": {
|
949
|
+
"success": True,
|
950
|
+
"namespace": "default",
|
951
|
+
"command_stats": [
|
952
|
+
{
|
953
|
+
"command": "GET",
|
954
|
+
"calls": 1500000,
|
955
|
+
"usec": 45000000,
|
956
|
+
"usec_per_call": 30.0
|
957
|
+
},
|
958
|
+
{
|
959
|
+
"command": "SET",
|
960
|
+
"calls": 800000,
|
961
|
+
"usec": 32000000,
|
962
|
+
"usec_per_call": 40.0
|
963
|
+
}
|
964
|
+
],
|
965
|
+
"total_commands": 25
|
966
|
+
}
|
967
|
+
}
|
968
|
+
}
|
969
|
+
},
|
970
|
+
500: {"description": "服务器内部错误"}
|
971
|
+
}
|
972
|
+
)
|
484
973
|
async def get_redis_command_stats(request: Request, namespace: str):
|
485
974
|
"""
|
486
|
-
获取Redis命令统计
|
487
|
-
|
975
|
+
## 获取 Redis 命令统计
|
976
|
+
|
977
|
+
获取 Redis 实例的命令执行统计信息,用于分析命令使用情况和性能优化。
|
978
|
+
|
979
|
+
**统计信息包括**:
|
980
|
+
- 命令名称
|
981
|
+
- 调用次数 (calls)
|
982
|
+
- 总耗时(微秒)(usec)
|
983
|
+
- 平均耗时(微秒/次)(usec_per_call)
|
984
|
+
|
985
|
+
**使用场景**:
|
986
|
+
- 分析 Redis 命令使用模式
|
987
|
+
- 识别高频命令
|
988
|
+
- 性能优化
|
989
|
+
- 命令耗时分析
|
990
|
+
- 容量规划
|
991
|
+
|
992
|
+
**示例请求**:
|
993
|
+
```bash
|
994
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/command-stats/default"
|
995
|
+
```
|
996
|
+
|
997
|
+
**返回示例**:
|
998
|
+
```json
|
999
|
+
{
|
1000
|
+
"success": true,
|
1001
|
+
"namespace": "default",
|
1002
|
+
"command_stats": [
|
1003
|
+
{"command": "GET", "calls": 1500000, "usec": 45000000, "usec_per_call": 30.0},
|
1004
|
+
{"command": "SET", "calls": 800000, "usec": 32000000, "usec_per_call": 40.0},
|
1005
|
+
{"command": "HGETALL", "calls": 50000, "usec": 8000000, "usec_per_call": 160.0}
|
1006
|
+
],
|
1007
|
+
"total_commands": 25
|
1008
|
+
}
|
1009
|
+
```
|
1010
|
+
|
1011
|
+
**注意事项**:
|
1012
|
+
- 统计数据为累计值,从 Redis 启动开始计算
|
1013
|
+
- 重启 Redis 后统计清零
|
1014
|
+
- 按调用次数降序排序
|
1015
|
+
- 可通过 `usec_per_call` 识别慢命令
|
1016
|
+
- 建议关注平均耗时较高的命令
|
1017
|
+
|
488
1018
|
Args:
|
489
1019
|
namespace: 命名空间
|
490
1020
|
"""
|
@@ -501,18 +1031,89 @@ async def get_redis_command_stats(request: Request, namespace: str):
|
|
501
1031
|
raise HTTPException(status_code=500, detail=str(e))
|
502
1032
|
|
503
1033
|
|
504
|
-
@router.get(
|
1034
|
+
@router.get(
|
1035
|
+
"/redis/stream-stats/{namespace}",
|
1036
|
+
summary="获取 Redis Stream 统计",
|
1037
|
+
description="获取 Redis Stream 的详细统计信息,包括长度、消费者组、消息等",
|
1038
|
+
responses={
|
1039
|
+
200: {
|
1040
|
+
"description": "成功返回 Stream 统计",
|
1041
|
+
"content": {
|
1042
|
+
"application/json": {
|
1043
|
+
"example": {
|
1044
|
+
"success": True,
|
1045
|
+
"namespace": "default",
|
1046
|
+
"streams": [
|
1047
|
+
{
|
1048
|
+
"stream_name": "task_stream",
|
1049
|
+
"length": 1500,
|
1050
|
+
"first_entry_id": "1697644800000-0",
|
1051
|
+
"last_entry_id": "1697731200000-5",
|
1052
|
+
"groups": [
|
1053
|
+
{
|
1054
|
+
"name": "workers",
|
1055
|
+
"consumers": 5,
|
1056
|
+
"pending": 120,
|
1057
|
+
"last_delivered_id": "1697730000000-3"
|
1058
|
+
}
|
1059
|
+
]
|
1060
|
+
}
|
1061
|
+
]
|
1062
|
+
}
|
1063
|
+
}
|
1064
|
+
}
|
1065
|
+
},
|
1066
|
+
500: {"description": "服务器内部错误"}
|
1067
|
+
}
|
1068
|
+
)
|
505
1069
|
async def get_redis_stream_stats(
|
506
1070
|
request: Request,
|
507
1071
|
namespace: str,
|
508
|
-
stream_name: Optional[str] = Query(None, description="Stream
|
1072
|
+
stream_name: Optional[str] = Query(None, description="Stream 名称,为空则返回所有 Stream 的统计", example="task_stream")
|
509
1073
|
):
|
510
1074
|
"""
|
511
|
-
获取Redis Stream统计
|
512
|
-
|
1075
|
+
## 获取 Redis Stream 统计
|
1076
|
+
|
1077
|
+
获取 Redis Stream 的详细统计信息,包括消息数量、消费者组状态等。
|
1078
|
+
|
1079
|
+
**Stream 统计信息包括**:
|
1080
|
+
- Stream 名称
|
1081
|
+
- 消息总数(length)
|
1082
|
+
- 第一条消息 ID
|
1083
|
+
- 最后一条消息 ID
|
1084
|
+
- 消费者组列表及其状态
|
1085
|
+
|
1086
|
+
**消费者组统计**:
|
1087
|
+
- 组名称
|
1088
|
+
- 消费者数量
|
1089
|
+
- 待处理消息数 (pending)
|
1090
|
+
- 最后交付的消息 ID
|
1091
|
+
|
1092
|
+
**使用场景**:
|
1093
|
+
- Stream 使用情况监控
|
1094
|
+
- 消费者组管理
|
1095
|
+
- 积压分析
|
1096
|
+
- 容量规划
|
1097
|
+
- 性能优化
|
1098
|
+
|
1099
|
+
**示例请求**:
|
1100
|
+
```bash
|
1101
|
+
# 获取所有 Stream 的统计
|
1102
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/stream-stats/default"
|
1103
|
+
|
1104
|
+
# 获取指定 Stream 的统计
|
1105
|
+
curl -X GET "http://localhost:8001/api/v1/queues/redis/stream-stats/default?stream_name=task_stream"
|
1106
|
+
```
|
1107
|
+
|
1108
|
+
**注意事项**:
|
1109
|
+
- 仅返回使用 Redis Stream 的队列统计
|
1110
|
+
- 消息 ID 格式为 `timestamp-sequence`
|
1111
|
+
- pending 表示已分配但未确认的消息数
|
1112
|
+
- 建议监控 pending 消息数,及时处理积压
|
1113
|
+
|
513
1114
|
Args:
|
514
1115
|
namespace: 命名空间
|
515
|
-
stream_name: 可选,指定
|
1116
|
+
stream_name: 可选,指定 Stream 名称
|
516
1117
|
"""
|
517
1118
|
try:
|
518
1119
|
app = request.app
|
@@ -527,4 +1128,107 @@ async def get_redis_stream_stats(
|
|
527
1128
|
raise HTTPException(status_code=500, detail=str(e))
|
528
1129
|
|
529
1130
|
|
1131
|
+
# ============ 任务管理 ============
|
1132
|
+
|
1133
|
+
@router.get(
|
1134
|
+
"/{namespace}/tasks",
|
1135
|
+
summary="获取队列任务列表(简化版)",
|
1136
|
+
description="获取指定队列的任务列表,向后兼容的简化版本",
|
1137
|
+
responses={
|
1138
|
+
200: {
|
1139
|
+
"description": "成功返回任务列表",
|
1140
|
+
"content": {
|
1141
|
+
"application/json": {
|
1142
|
+
"example": {
|
1143
|
+
"success": True,
|
1144
|
+
"data": [
|
1145
|
+
{
|
1146
|
+
"task_id": "task-001",
|
1147
|
+
"queue_name": "email_queue",
|
1148
|
+
"status": "completed",
|
1149
|
+
"created_at": "2025-10-18T10:00:00Z"
|
1150
|
+
}
|
1151
|
+
],
|
1152
|
+
"total": 50
|
1153
|
+
}
|
1154
|
+
}
|
1155
|
+
}
|
1156
|
+
},
|
1157
|
+
500: {"description": "服务器内部错误"}
|
1158
|
+
}
|
1159
|
+
)
|
1160
|
+
async def get_queue_tasks_simple(
|
1161
|
+
request: Request,
|
1162
|
+
namespace: str,
|
1163
|
+
queue_name: str = Query(..., description="队列名称", example="email_queue"),
|
1164
|
+
start_time: Optional[str] = Query(None, description="开始时间(ISO格式或 \"-\" 表示最早)", example="2025-10-18T00:00:00Z"),
|
1165
|
+
end_time: Optional[str] = Query(None, description="结束时间(ISO格式或 \"+\" 表示最新)", example="2025-10-18T23:59:59Z"),
|
1166
|
+
limit: int = Query(50, ge=1, le=1000, description="返回数量限制,范围 1-1000", example=50)
|
1167
|
+
):
|
1168
|
+
"""
|
1169
|
+
## 获取队列任务列表(简化版)
|
1170
|
+
|
1171
|
+
获取指定队列的任务列表,这是一个简化版本的API,用于向后兼容。
|
1172
|
+
|
1173
|
+
**功能特点**:
|
1174
|
+
- ✅ 简单的时间范围查询
|
1175
|
+
- ✅ 支持分页限制
|
1176
|
+
- ✅ 按时间倒序返回(最新的在前)
|
1177
|
+
- ✅ 向后兼容旧版本
|
1178
|
+
|
1179
|
+
**查询参数**:
|
1180
|
+
- `queue_name`: 队列名称(必填)
|
1181
|
+
- `start_time`: 开始时间,可选,默认 "-" 表示最早
|
1182
|
+
- `end_time`: 结束时间,可选,默认 "+" 表示最新
|
1183
|
+
- `limit`: 返回数量限制
|
1184
|
+
|
1185
|
+
**使用场景**:
|
1186
|
+
- 快速查看队列任务
|
1187
|
+
- 简单的任务列表查询
|
1188
|
+
- 旧版本API兼容
|
1189
|
+
|
1190
|
+
**示例请求**:
|
1191
|
+
```bash
|
1192
|
+
# 获取队列最近50条任务
|
1193
|
+
curl -X GET "http://localhost:8001/api/v1/queues/default/tasks?queue_name=email_queue&limit=50"
|
1194
|
+
|
1195
|
+
# 获取指定时间范围的任务
|
1196
|
+
curl -X GET "http://localhost:8001/api/v1/queues/default/tasks?queue_name=email_queue&start_time=2025-10-18T00:00:00Z&end_time=2025-10-18T23:59:59Z&limit=100"
|
1197
|
+
```
|
1198
|
+
|
1199
|
+
**注意事项**:
|
1200
|
+
- 默认从新到旧排序(reverse=True)
|
1201
|
+
- 如需更复杂的查询,请使用 `POST /tasks-v2/{namespace}` 接口
|
1202
|
+
- 时间参数支持 ISO 8601 格式或 "-"/"+" 特殊值
|
1203
|
+
- 最大返回1000条记录
|
1204
|
+
|
1205
|
+
Args:
|
1206
|
+
namespace: 命名空间
|
1207
|
+
queue_name: 队列名称
|
1208
|
+
start_time: 开始时间(可选)
|
1209
|
+
end_time: 结束时间(可选)
|
1210
|
+
limit: 返回数量限制
|
1211
|
+
|
1212
|
+
Returns:
|
1213
|
+
任务列表
|
1214
|
+
"""
|
1215
|
+
try:
|
1216
|
+
if not hasattr(request.app.state, 'monitor'):
|
1217
|
+
raise HTTPException(status_code=500, detail="Monitor service not initialized")
|
1218
|
+
|
1219
|
+
monitor = request.app.state.monitor
|
1220
|
+
result = await monitor.get_queue_tasks(
|
1221
|
+
queue_name,
|
1222
|
+
start_time or "-",
|
1223
|
+
end_time or "+",
|
1224
|
+
limit,
|
1225
|
+
reverse=True # 默认从新到旧
|
1226
|
+
)
|
1227
|
+
|
1228
|
+
return result
|
1229
|
+
except Exception as e:
|
1230
|
+
logger.error(f"获取队列 {queue_name} 任务列表失败: {e}", exc_info=True)
|
1231
|
+
raise HTTPException(status_code=500, detail=str(e))
|
1232
|
+
|
1233
|
+
|
530
1234
|
__all__ = ['router']
|