jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/constants.py +213 -0
- jettask/core/app.py +525 -205
- jettask/core/cli.py +193 -185
- jettask/core/consumer_manager.py +126 -34
- jettask/core/context.py +3 -0
- jettask/core/enums.py +137 -0
- jettask/core/event_pool.py +501 -168
- jettask/core/message.py +147 -0
- jettask/core/offline_worker_recovery.py +181 -114
- jettask/core/task.py +10 -174
- jettask/core/task_batch.py +153 -0
- jettask/core/unified_manager_base.py +243 -0
- jettask/core/worker_scanner.py +54 -54
- jettask/executors/asyncio.py +184 -64
- jettask/webui/backend/config.py +51 -0
- jettask/webui/backend/data_access.py +2083 -92
- jettask/webui/backend/data_api.py +3294 -0
- jettask/webui/backend/dependencies.py +261 -0
- jettask/webui/backend/init_meta_db.py +158 -0
- jettask/webui/backend/main.py +1358 -69
- jettask/webui/backend/main_unified.py +78 -0
- jettask/webui/backend/main_v2.py +394 -0
- jettask/webui/backend/namespace_api.py +295 -0
- jettask/webui/backend/namespace_api_old.py +294 -0
- jettask/webui/backend/namespace_data_access.py +611 -0
- jettask/webui/backend/queue_backlog_api.py +727 -0
- jettask/webui/backend/queue_stats_v2.py +521 -0
- jettask/webui/backend/redis_monitor_api.py +476 -0
- jettask/webui/backend/unified_api_router.py +1601 -0
- jettask/webui/db_init.py +204 -32
- jettask/webui/frontend/package-lock.json +492 -1
- jettask/webui/frontend/package.json +4 -1
- jettask/webui/frontend/src/App.css +105 -7
- jettask/webui/frontend/src/App.jsx +49 -20
- jettask/webui/frontend/src/components/NamespaceSelector.jsx +166 -0
- jettask/webui/frontend/src/components/QueueBacklogChart.jsx +298 -0
- jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +638 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.css +65 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.jsx +487 -0
- jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +465 -0
- jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +423 -0
- jettask/webui/frontend/src/components/TaskFilter.jsx +425 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.css +21 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.jsx +160 -0
- jettask/webui/frontend/src/components/layout/AppLayout.css +95 -0
- jettask/webui/frontend/src/components/layout/AppLayout.jsx +49 -0
- jettask/webui/frontend/src/components/layout/Header.css +34 -10
- jettask/webui/frontend/src/components/layout/Header.jsx +31 -23
- jettask/webui/frontend/src/components/layout/SideMenu.css +137 -0
- jettask/webui/frontend/src/components/layout/SideMenu.jsx +209 -0
- jettask/webui/frontend/src/components/layout/TabsNav.css +244 -0
- jettask/webui/frontend/src/components/layout/TabsNav.jsx +206 -0
- jettask/webui/frontend/src/components/layout/UserInfo.css +197 -0
- jettask/webui/frontend/src/components/layout/UserInfo.jsx +197 -0
- jettask/webui/frontend/src/contexts/NamespaceContext.jsx +72 -0
- jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +245 -0
- jettask/webui/frontend/src/main.jsx +1 -0
- jettask/webui/frontend/src/pages/Alerts.jsx +684 -0
- jettask/webui/frontend/src/pages/Dashboard.jsx +1330 -0
- jettask/webui/frontend/src/pages/QueueDetail.jsx +1109 -10
- jettask/webui/frontend/src/pages/QueueMonitor.jsx +236 -115
- jettask/webui/frontend/src/pages/Queues.jsx +5 -1
- jettask/webui/frontend/src/pages/ScheduledTasks.jsx +809 -0
- jettask/webui/frontend/src/pages/Settings.jsx +800 -0
- jettask/webui/frontend/src/services/api.js +7 -5
- jettask/webui/frontend/src/utils/suppressWarnings.js +22 -0
- jettask/webui/frontend/src/utils/userPreferences.js +154 -0
- jettask/webui/multi_namespace_consumer.py +543 -0
- jettask/webui/pg_consumer.py +983 -246
- jettask/webui/static/dist/assets/index-7129cfe1.css +1 -0
- jettask/webui/static/dist/assets/index-8d1935cc.js +774 -0
- jettask/webui/static/dist/index.html +2 -2
- jettask/webui/task_center.py +216 -0
- jettask/webui/task_center_client.py +150 -0
- jettask/webui/unified_consumer_manager.py +193 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/METADATA +1 -1
- jettask-0.2.4.dist-info/RECORD +134 -0
- jettask/webui/pg_consumer_slow.py +0 -1099
- jettask/webui/pg_consumer_test.py +0 -678
- jettask/webui/static/dist/assets/index-823408e8.css +0 -1
- jettask/webui/static/dist/assets/index-9968b0b8.js +0 -543
- jettask/webui/test_pg_consumer_recovery.py +0 -547
- jettask/webui/test_recovery_simple.py +0 -492
- jettask/webui/test_self_recovery.py +0 -467
- jettask-0.2.1.dist-info/RECORD +0 -91
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/WHEEL +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/top_level.txt +0 -0
@@ -1,467 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
"""测试单个worker重启后自动恢复pending消息的机制"""
|
3
|
-
|
4
|
-
import asyncio
|
5
|
-
import json
|
6
|
-
import logging
|
7
|
-
import os
|
8
|
-
import sys
|
9
|
-
import time
|
10
|
-
from typing import Dict, List, Optional
|
11
|
-
from datetime import datetime, timezone
|
12
|
-
|
13
|
-
import redis.asyncio as redis
|
14
|
-
from redis.asyncio import Redis
|
15
|
-
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
16
|
-
from sqlalchemy.orm import sessionmaker
|
17
|
-
from sqlalchemy import text
|
18
|
-
|
19
|
-
from jettask.webui.config import PostgreSQLConfig, RedisConfig
|
20
|
-
from jettask.utils.serializer import dumps_str
|
21
|
-
|
22
|
-
logger = logging.getLogger(__name__)
|
23
|
-
logging.basicConfig(
|
24
|
-
level=logging.INFO,
|
25
|
-
format='%(asctime)s - %(levelname)s - %(message)s'
|
26
|
-
)
|
27
|
-
|
28
|
-
class SelfRecoveryTester:
|
29
|
-
"""测试单个worker自我恢复机制"""
|
30
|
-
|
31
|
-
def __init__(self):
|
32
|
-
self.pg_config = PostgreSQLConfig(
|
33
|
-
host=os.getenv('JETTASK_PG_HOST', 'localhost'),
|
34
|
-
port=int(os.getenv('JETTASK_PG_PORT', '5432')),
|
35
|
-
database=os.getenv('JETTASK_PG_DB', 'jettask'),
|
36
|
-
user=os.getenv('JETTASK_PG_USER', 'jettask'),
|
37
|
-
password=os.getenv('JETTASK_PG_PASSWORD', '123456'),
|
38
|
-
)
|
39
|
-
|
40
|
-
self.redis_config = RedisConfig(
|
41
|
-
host=os.getenv('REDIS_HOST', 'localhost'),
|
42
|
-
port=int(os.getenv('REDIS_PORT', '6379')),
|
43
|
-
db=int(os.getenv('REDIS_DB', '0')),
|
44
|
-
password=os.getenv('REDIS_PASSWORD'),
|
45
|
-
)
|
46
|
-
|
47
|
-
self.prefix = "jettask"
|
48
|
-
self.redis_client: Optional[Redis] = None
|
49
|
-
|
50
|
-
async def setup(self):
|
51
|
-
"""初始化连接"""
|
52
|
-
self.redis_client = await redis.Redis(
|
53
|
-
host=self.redis_config.host,
|
54
|
-
port=self.redis_config.port,
|
55
|
-
db=self.redis_config.db,
|
56
|
-
password=self.redis_config.password,
|
57
|
-
decode_responses=False
|
58
|
-
)
|
59
|
-
|
60
|
-
async def cleanup(self):
|
61
|
-
"""清理连接"""
|
62
|
-
if self.redis_client:
|
63
|
-
await self.redis_client.aclose()
|
64
|
-
|
65
|
-
async def test_queue_self_recovery(self):
|
66
|
-
"""测试队列消息的自我恢复机制"""
|
67
|
-
logger.info("=" * 60)
|
68
|
-
logger.info("测试单个worker重启后自动恢复自己的pending消息")
|
69
|
-
logger.info("=" * 60)
|
70
|
-
|
71
|
-
queue_name = 'SELF_RECOVERY_QUEUE'
|
72
|
-
stream_key = f"{self.prefix}:QUEUE:{queue_name}"
|
73
|
-
consumer_group = f"{self.prefix}_pg_consumer1"
|
74
|
-
consumer_name = "self_recovery_worker"
|
75
|
-
|
76
|
-
# 1. 清理旧数据
|
77
|
-
logger.info("\n1. 清理旧数据...")
|
78
|
-
try:
|
79
|
-
await self.redis_client.delete(stream_key)
|
80
|
-
except:
|
81
|
-
pass
|
82
|
-
|
83
|
-
# 2. 发送测试消息
|
84
|
-
logger.info("\n2. 发送测试消息到队列...")
|
85
|
-
message_ids = []
|
86
|
-
for i in range(5):
|
87
|
-
task_data = {
|
88
|
-
'name': f'self_recovery_task_{i}',
|
89
|
-
'queue': queue_name,
|
90
|
-
'priority': 1,
|
91
|
-
'trigger_time': time.time(),
|
92
|
-
'test_id': f'self_recovery_{i}'
|
93
|
-
}
|
94
|
-
|
95
|
-
msg_id = await self.redis_client.xadd(
|
96
|
-
stream_key,
|
97
|
-
{'data': dumps_str(task_data)}
|
98
|
-
)
|
99
|
-
message_ids.append(msg_id)
|
100
|
-
logger.info(f" - 发送消息 {i}: {msg_id}")
|
101
|
-
|
102
|
-
# 3. 创建消费者组
|
103
|
-
logger.info("\n3. 创建消费者组...")
|
104
|
-
try:
|
105
|
-
await self.redis_client.xgroup_create(
|
106
|
-
stream_key, consumer_group, id='0', mkstream=True
|
107
|
-
)
|
108
|
-
logger.info(f" - 创建消费者组: {consumer_group}")
|
109
|
-
except:
|
110
|
-
pass
|
111
|
-
|
112
|
-
# 4. 模拟worker第一次读取消息(读取新消息)
|
113
|
-
logger.info(f"\n4. 模拟 {consumer_name} 第一次读取新消息但不ACK(模拟突然挂掉)...")
|
114
|
-
|
115
|
-
messages = await self.redis_client.xreadgroup(
|
116
|
-
consumer_group,
|
117
|
-
consumer_name,
|
118
|
-
{stream_key: '>'}, # '>' 表示读取新消息
|
119
|
-
count=5,
|
120
|
-
block=1000
|
121
|
-
)
|
122
|
-
|
123
|
-
first_read_ids = []
|
124
|
-
if messages:
|
125
|
-
logger.info(f" - {consumer_name} 读取了 {len(messages[0][1])} 条新消息")
|
126
|
-
for msg_id, data in messages[0][1]:
|
127
|
-
first_read_ids.append(msg_id)
|
128
|
-
logger.info(f" - 消息ID: {msg_id}")
|
129
|
-
|
130
|
-
# 5. 检查pending消息
|
131
|
-
logger.info("\n5. 检查pending消息状态...")
|
132
|
-
pending_info = await self.redis_client.xpending(stream_key, consumer_group)
|
133
|
-
logger.info(f" - 总pending消息数: {pending_info['pending']}")
|
134
|
-
|
135
|
-
if pending_info['pending'] > 0:
|
136
|
-
detailed = await self.redis_client.xpending_range(
|
137
|
-
stream_key, consumer_group,
|
138
|
-
min='-', max='+', count=10
|
139
|
-
)
|
140
|
-
for msg in detailed:
|
141
|
-
logger.info(f" - 消息 {msg['message_id']}: consumer={msg['consumer']}, times_delivered={msg['times_delivered']}")
|
142
|
-
|
143
|
-
# 6. 模拟同一个worker重启后读取pending消息(不使用XCLAIM)
|
144
|
-
logger.info(f"\n6. 模拟 {consumer_name} 重启后读取自己的pending消息...")
|
145
|
-
logger.info(" - 使用 '0' 作为起始ID,这会让consumer读取自己的pending消息")
|
146
|
-
|
147
|
-
# 先读取pending消息(使用0作为起始ID)
|
148
|
-
pending_messages = await self.redis_client.xreadgroup(
|
149
|
-
consumer_group,
|
150
|
-
consumer_name, # 同一个consumer name
|
151
|
-
{stream_key: '0'}, # '0' 或 '0-0' 表示从头读取pending消息
|
152
|
-
count=10,
|
153
|
-
block=0 # 不阻塞
|
154
|
-
)
|
155
|
-
|
156
|
-
recovered_ids = []
|
157
|
-
if pending_messages:
|
158
|
-
logger.info(f" - {consumer_name} 成功读取到 {len(pending_messages[0][1])} 条自己的pending消息")
|
159
|
-
for msg_id, data in pending_messages[0][1]:
|
160
|
-
recovered_ids.append(msg_id)
|
161
|
-
logger.info(f" - 恢复消息ID: {msg_id}")
|
162
|
-
|
163
|
-
# ACK这些消息
|
164
|
-
if recovered_ids:
|
165
|
-
await self.redis_client.xack(stream_key, consumer_group, *recovered_ids)
|
166
|
-
logger.info(f" - {consumer_name} ACK了 {len(recovered_ids)} 条恢复的消息")
|
167
|
-
else:
|
168
|
-
logger.info(f" - {consumer_name} 没有读取到pending消息")
|
169
|
-
|
170
|
-
# 7. 再次检查pending消息
|
171
|
-
logger.info("\n7. 再次检查pending消息状态...")
|
172
|
-
pending_info_after = await self.redis_client.xpending(stream_key, consumer_group)
|
173
|
-
logger.info(f" - 总pending消息数: {pending_info_after['pending']}")
|
174
|
-
|
175
|
-
# 8. 验证结果
|
176
|
-
logger.info("\n" + "=" * 60)
|
177
|
-
if pending_info['pending'] > 0 and pending_info_after['pending'] == 0:
|
178
|
-
logger.info("✓ 单个worker自我恢复测试通过!")
|
179
|
-
logger.info(" worker重启后成功读取并处理了自己的pending消息,无需XCLAIM")
|
180
|
-
return True
|
181
|
-
else:
|
182
|
-
logger.warning(f"✗ 单个worker自我恢复测试失败。")
|
183
|
-
logger.warning(f" 恢复前pending: {pending_info['pending']},恢复后pending: {pending_info_after['pending']}")
|
184
|
-
return False
|
185
|
-
|
186
|
-
async def test_task_changes_self_recovery(self):
|
187
|
-
"""测试TASK_CHANGES的自我恢复机制"""
|
188
|
-
logger.info("\n" + "=" * 60)
|
189
|
-
logger.info("测试TASK_CHANGES单个worker自我恢复机制")
|
190
|
-
logger.info("=" * 60)
|
191
|
-
|
192
|
-
change_stream_key = f"{self.prefix}:TASK_CHANGES"
|
193
|
-
consumer_group = f"{self.prefix}_changes_consumer"
|
194
|
-
consumer_name = "changes_self_recovery_worker"
|
195
|
-
|
196
|
-
# 1. 清理旧数据
|
197
|
-
logger.info("\n1. 清理旧数据...")
|
198
|
-
try:
|
199
|
-
await self.redis_client.delete(change_stream_key)
|
200
|
-
except:
|
201
|
-
pass
|
202
|
-
|
203
|
-
# 2. 发送任务变更事件
|
204
|
-
logger.info("\n2. 发送任务变更事件...")
|
205
|
-
event_ids = []
|
206
|
-
for i in range(5):
|
207
|
-
event_data = {
|
208
|
-
'event_id': f'self_recovery_change_{i}',
|
209
|
-
'event_type': 'task_updated',
|
210
|
-
'timestamp': str(time.time())
|
211
|
-
}
|
212
|
-
|
213
|
-
msg_id = await self.redis_client.xadd(
|
214
|
-
change_stream_key,
|
215
|
-
event_data
|
216
|
-
)
|
217
|
-
event_ids.append(msg_id)
|
218
|
-
logger.info(f" - 发送事件 {i}: {msg_id}")
|
219
|
-
|
220
|
-
# 3. 创建消费者组
|
221
|
-
logger.info("\n3. 创建消费者组...")
|
222
|
-
try:
|
223
|
-
await self.redis_client.xgroup_create(
|
224
|
-
change_stream_key, consumer_group, id='0', mkstream=True
|
225
|
-
)
|
226
|
-
logger.info(f" - 创建消费者组: {consumer_group}")
|
227
|
-
except:
|
228
|
-
pass
|
229
|
-
|
230
|
-
# 4. 模拟worker第一次读取消息但不ACK
|
231
|
-
logger.info(f"\n4. 模拟 {consumer_name} 第一次读取事件但不ACK...")
|
232
|
-
|
233
|
-
messages = await self.redis_client.xreadgroup(
|
234
|
-
consumer_group,
|
235
|
-
consumer_name,
|
236
|
-
{change_stream_key: '>'},
|
237
|
-
count=5,
|
238
|
-
block=1000
|
239
|
-
)
|
240
|
-
|
241
|
-
if messages:
|
242
|
-
logger.info(f" - {consumer_name} 读取了 {len(messages[0][1])} 条事件")
|
243
|
-
|
244
|
-
# 5. 检查pending事件
|
245
|
-
logger.info("\n5. 检查pending事件...")
|
246
|
-
pending_info = await self.redis_client.xpending(change_stream_key, consumer_group)
|
247
|
-
logger.info(f" - 总pending事件数: {pending_info['pending']}")
|
248
|
-
|
249
|
-
# 6. 模拟同一个worker重启后恢复
|
250
|
-
logger.info(f"\n6. 模拟 {consumer_name} 重启后读取自己的pending事件...")
|
251
|
-
|
252
|
-
# 读取pending消息
|
253
|
-
pending_messages = await self.redis_client.xreadgroup(
|
254
|
-
consumer_group,
|
255
|
-
consumer_name,
|
256
|
-
{change_stream_key: '0'}, # 从头读取pending
|
257
|
-
count=10,
|
258
|
-
block=0
|
259
|
-
)
|
260
|
-
|
261
|
-
recovered_ids = []
|
262
|
-
if pending_messages:
|
263
|
-
logger.info(f" - {consumer_name} 成功恢复 {len(pending_messages[0][1])} 条pending事件")
|
264
|
-
for msg_id, data in pending_messages[0][1]:
|
265
|
-
recovered_ids.append(msg_id)
|
266
|
-
|
267
|
-
# ACK消息
|
268
|
-
if recovered_ids:
|
269
|
-
await self.redis_client.xack(change_stream_key, consumer_group, *recovered_ids)
|
270
|
-
logger.info(f" - {consumer_name} ACK了 {len(recovered_ids)} 条恢复的事件")
|
271
|
-
|
272
|
-
# 7. 再次检查pending事件
|
273
|
-
logger.info("\n7. 再次检查pending事件...")
|
274
|
-
pending_info_after = await self.redis_client.xpending(change_stream_key, consumer_group)
|
275
|
-
logger.info(f" - 总pending事件数: {pending_info_after['pending']}")
|
276
|
-
|
277
|
-
# 8. 验证结果
|
278
|
-
logger.info("\n" + "=" * 60)
|
279
|
-
if pending_info['pending'] > 0 and pending_info_after['pending'] == 0:
|
280
|
-
logger.info("✓ TASK_CHANGES自我恢复测试通过!")
|
281
|
-
return True
|
282
|
-
else:
|
283
|
-
logger.warning(f"✗ TASK_CHANGES自我恢复测试失败")
|
284
|
-
return False
|
285
|
-
|
286
|
-
async def demonstrate_pg_consumer_recovery_flow(self):
|
287
|
-
"""演示pg_consumer实际的恢复流程"""
|
288
|
-
logger.info("\n" + "=" * 60)
|
289
|
-
logger.info("演示pg_consumer的实际恢复流程")
|
290
|
-
logger.info("=" * 60)
|
291
|
-
|
292
|
-
queue_name = 'PG_CONSUMER_DEMO'
|
293
|
-
stream_key = f"{self.prefix}:QUEUE:{queue_name}"
|
294
|
-
consumer_group = f"{self.prefix}_pg_consumer1"
|
295
|
-
consumer_name = "pg_consumer_worker_1"
|
296
|
-
|
297
|
-
# 清理
|
298
|
-
try:
|
299
|
-
await self.redis_client.delete(stream_key)
|
300
|
-
except:
|
301
|
-
pass
|
302
|
-
|
303
|
-
# 发送消息
|
304
|
-
logger.info("\n1. 发送5条消息...")
|
305
|
-
for i in range(5):
|
306
|
-
await self.redis_client.xadd(
|
307
|
-
stream_key,
|
308
|
-
{'data': dumps_str({'name': f'demo_task_{i}'})}
|
309
|
-
)
|
310
|
-
|
311
|
-
# 创建消费者组
|
312
|
-
try:
|
313
|
-
await self.redis_client.xgroup_create(
|
314
|
-
stream_key, consumer_group, id='0', mkstream=True
|
315
|
-
)
|
316
|
-
except:
|
317
|
-
pass
|
318
|
-
|
319
|
-
# 模拟pg_consumer的消费逻辑
|
320
|
-
logger.info(f"\n2. {consumer_name} 开始消费...")
|
321
|
-
|
322
|
-
check_backlog = True
|
323
|
-
lastid = "0-0"
|
324
|
-
processed_count = 0
|
325
|
-
|
326
|
-
while True:
|
327
|
-
# 这是pg_consumer的实际逻辑
|
328
|
-
myid = lastid if check_backlog else ">"
|
329
|
-
|
330
|
-
logger.info(f" - 读取消息,myid={myid}, check_backlog={check_backlog}")
|
331
|
-
|
332
|
-
messages = await self.redis_client.xreadgroup(
|
333
|
-
consumer_group,
|
334
|
-
consumer_name,
|
335
|
-
{stream_key: myid},
|
336
|
-
count=2, # 批量读取
|
337
|
-
block=0 if check_backlog else 1000
|
338
|
-
)
|
339
|
-
|
340
|
-
if not messages or len(messages[0][1]) == 0:
|
341
|
-
if check_backlog:
|
342
|
-
logger.info(" - 没有更多backlog消息,切换到读取新消息模式")
|
343
|
-
check_backlog = False
|
344
|
-
continue
|
345
|
-
else:
|
346
|
-
logger.info(" - 没有新消息")
|
347
|
-
break
|
348
|
-
|
349
|
-
# 处理消息
|
350
|
-
msg_count = len(messages[0][1])
|
351
|
-
logger.info(f" - 读取到 {msg_count} 条消息")
|
352
|
-
|
353
|
-
# 模拟处理失败(第一次只处理2条就"挂掉")
|
354
|
-
if processed_count < 2:
|
355
|
-
processed_count += msg_count
|
356
|
-
if processed_count >= 2:
|
357
|
-
logger.info(" - 模拟worker挂掉,不ACK剩余消息")
|
358
|
-
break
|
359
|
-
|
360
|
-
# 正常ACK
|
361
|
-
msg_ids = [msg[0] for msg in messages[0][1]]
|
362
|
-
await self.redis_client.xack(stream_key, consumer_group, *msg_ids)
|
363
|
-
logger.info(f" - ACK了 {len(msg_ids)} 条消息")
|
364
|
-
|
365
|
-
# 更新lastid
|
366
|
-
lastid = messages[0][1][-1][0].decode('utf-8') if isinstance(messages[0][1][-1][0], bytes) else messages[0][1][-1][0]
|
367
|
-
|
368
|
-
# 检查是否还有更多backlog
|
369
|
-
check_backlog = msg_count >= 2
|
370
|
-
|
371
|
-
# 检查pending
|
372
|
-
pending_info = await self.redis_client.xpending(stream_key, consumer_group)
|
373
|
-
logger.info(f"\n3. Worker挂掉后,pending消息数: {pending_info['pending']}")
|
374
|
-
|
375
|
-
# 模拟重启后恢复
|
376
|
-
logger.info(f"\n4. {consumer_name} 重启,继续消费...")
|
377
|
-
|
378
|
-
check_backlog = True
|
379
|
-
lastid = "0-0" # 重新从0开始,会读取pending消息
|
380
|
-
|
381
|
-
while True:
|
382
|
-
myid = lastid if check_backlog else ">"
|
383
|
-
|
384
|
-
logger.info(f" - 读取消息,myid={myid}, check_backlog={check_backlog}")
|
385
|
-
|
386
|
-
messages = await self.redis_client.xreadgroup(
|
387
|
-
consumer_group,
|
388
|
-
consumer_name,
|
389
|
-
{stream_key: myid},
|
390
|
-
count=10,
|
391
|
-
block=0 if check_backlog else 1000
|
392
|
-
)
|
393
|
-
|
394
|
-
if not messages or len(messages[0][1]) == 0:
|
395
|
-
if check_backlog:
|
396
|
-
logger.info(" - 没有更多pending/backlog消息")
|
397
|
-
check_backlog = False
|
398
|
-
continue
|
399
|
-
else:
|
400
|
-
logger.info(" - 没有新消息,完成")
|
401
|
-
break
|
402
|
-
|
403
|
-
msg_count = len(messages[0][1])
|
404
|
-
logger.info(f" - 恢复并处理 {msg_count} 条消息")
|
405
|
-
|
406
|
-
# ACK消息
|
407
|
-
msg_ids = [msg[0] for msg in messages[0][1]]
|
408
|
-
await self.redis_client.xack(stream_key, consumer_group, *msg_ids)
|
409
|
-
logger.info(f" - ACK了 {len(msg_ids)} 条消息")
|
410
|
-
|
411
|
-
# 更新lastid
|
412
|
-
lastid = messages[0][1][-1][0].decode('utf-8') if isinstance(messages[0][1][-1][0], bytes) else messages[0][1][-1][0]
|
413
|
-
check_backlog = msg_count >= 10
|
414
|
-
|
415
|
-
# 最终检查
|
416
|
-
pending_info_after = await self.redis_client.xpending(stream_key, consumer_group)
|
417
|
-
logger.info(f"\n5. 恢复完成后,pending消息数: {pending_info_after['pending']}")
|
418
|
-
|
419
|
-
logger.info("\n" + "=" * 60)
|
420
|
-
logger.info("✓ 演示完成:pg_consumer使用check_backlog机制自动恢复pending消息")
|
421
|
-
logger.info(" 关键点:重启后设置lastid='0-0'和check_backlog=True")
|
422
|
-
logger.info(" 这样会先处理所有pending和未读消息,然后再读取新消息")
|
423
|
-
|
424
|
-
|
425
|
-
async def main():
|
426
|
-
"""主测试函数"""
|
427
|
-
from dotenv import load_dotenv
|
428
|
-
load_dotenv()
|
429
|
-
|
430
|
-
tester = SelfRecoveryTester()
|
431
|
-
await tester.setup()
|
432
|
-
|
433
|
-
try:
|
434
|
-
# 测试1:队列消息自我恢复
|
435
|
-
queue_test = await tester.test_queue_self_recovery()
|
436
|
-
await asyncio.sleep(1)
|
437
|
-
|
438
|
-
# 测试2:TASK_CHANGES自我恢复
|
439
|
-
changes_test = await tester.test_task_changes_self_recovery()
|
440
|
-
await asyncio.sleep(1)
|
441
|
-
|
442
|
-
# 演示pg_consumer的实际恢复流程
|
443
|
-
await tester.demonstrate_pg_consumer_recovery_flow()
|
444
|
-
|
445
|
-
# 总结
|
446
|
-
logger.info("\n" + "=" * 60)
|
447
|
-
logger.info("测试总结")
|
448
|
-
logger.info("=" * 60)
|
449
|
-
logger.info(f"队列消息自我恢复测试: {'✓ 通过' if queue_test else '✗ 失败'}")
|
450
|
-
logger.info(f"TASK_CHANGES自我恢复测试: {'✓ 通过' if changes_test else '✗ 失败'}")
|
451
|
-
|
452
|
-
if queue_test and changes_test:
|
453
|
-
logger.info("\n🎉 所有测试通过!")
|
454
|
-
logger.info("\n关键发现:")
|
455
|
-
logger.info("1. 单个worker重启后,使用相同的consumer_name读取")
|
456
|
-
logger.info("2. 设置起始ID为'0'或'0-0',可以读取自己的pending消息")
|
457
|
-
logger.info("3. 不需要XCLAIM,直接xreadgroup即可恢复")
|
458
|
-
logger.info("4. pg_consumer的check_backlog机制完美支持这种恢复方式")
|
459
|
-
|
460
|
-
except Exception as e:
|
461
|
-
logger.error(f"测试出错: {e}", exc_info=True)
|
462
|
-
finally:
|
463
|
-
await tester.cleanup()
|
464
|
-
|
465
|
-
|
466
|
-
if __name__ == '__main__':
|
467
|
-
asyncio.run(main())
|
jettask-0.2.1.dist-info/RECORD
DELETED
@@ -1,91 +0,0 @@
|
|
1
|
-
jettask/__init__.py,sha256=uEpevpMSC75jMQ8wNca9oFJcoJ8LjyOPqD0AVwleba0,1133
|
2
|
-
jettask/exceptions.py,sha256=aQTUpiF2ftiZwGORUHXatAKbDEzptUf6iP7ZOXBQ3tQ,1971
|
3
|
-
jettask/router.py,sha256=RWlaQKDeQIHqqshT026wGKMLk3HtnqM_8L6d7MgPP_4,5177
|
4
|
-
jettask/config/__init__.py,sha256=qCRGmiXSK45LDU9pr0bUC-VoZAkTK1jAch1i9tvHCeE,158
|
5
|
-
jettask/config/performance.py,sha256=bOdLEskfB_6cRfS10IRgmtKEsJw_CaIZsPHbXxaHwbU,5686
|
6
|
-
jettask/core/__init__.py,sha256=CvBoBCERXCo-jgnkPqAuIgT4uC7oQMnSi7okRxMi6Vc,181
|
7
|
-
jettask/core/app.py,sha256=06_uJ6mBMxqrrSexYqd_gz5ydFTOiHKI3z7j2kL1hDk,59936
|
8
|
-
jettask/core/app_importer.py,sha256=B8WiSUz5_O5jlFIBr1CxI_F2gqFYK6ItpONiY_4AiXI,10266
|
9
|
-
jettask/core/cli.py,sha256=vO7aBtKgUzGpI737Ty6d1Oio48_U_8_KhvBXcEEmafE,18139
|
10
|
-
jettask/core/consumer_manager.py,sha256=UaXOMuosPjrEZ_RR734bLihbyKb04qPqAAV3_O5cKk4,76020
|
11
|
-
jettask/core/context.py,sha256=JrDYmYEiowBbw4uIChcz2kfYdOSOliR2SCR66jNz40c,789
|
12
|
-
jettask/core/delay_scanner.py,sha256=rwbIA7SFyOxAV14FW7NB40_djFrrqJJpNkpOri7XcZI,8394
|
13
|
-
jettask/core/event_pool.py,sha256=jKRf375F7eyG6yQqLhszJu1GWmqlwc_wtf-M2ZtdFuk,68231
|
14
|
-
jettask/core/heartbeat_process.py,sha256=G-4rHSMgHIvFCRPlZeh91tKd0Fir8jJ9YVL4vdajQDE,9233
|
15
|
-
jettask/core/offline_worker_recovery.py,sha256=tHGjCYBbsyYiTw33XS-UbVGBZPPD2ME6tqsn8OjE_uM,19820
|
16
|
-
jettask/core/retry.py,sha256=lX56Z0fvKjEhVbYPHYdoN-W4XUPVplHlfIOupBylEeI,10711
|
17
|
-
jettask/core/task.py,sha256=5nTOQABIP1pcXs4vK9ypoV5StJaZ71WLKH2PmqcPg50,12072
|
18
|
-
jettask/core/worker_scanner.py,sha256=dXLGffolMLixG0PWei0i4xyD3BDe3abP01VQXOAULHM,11169
|
19
|
-
jettask/executors/__init__.py,sha256=1aXRvwHzlypVcHNYy60UgUELOlNmbPp8oFMS5T0tJbs,186
|
20
|
-
jettask/executors/asyncio.py,sha256=oI0yNe2uZH3eUyYncyUi5b5j-Aq6UvZcvafGj5F01y0,37036
|
21
|
-
jettask/executors/base.py,sha256=XwTIb8-_pTr_C3skRBOy9qgVpGOlFux258BNKG6CP6k,776
|
22
|
-
jettask/executors/common.py,sha256=eLYtBSzh44hHEiJAHFXtNyIMucsOTc2O-IZJt143OYg,6246
|
23
|
-
jettask/executors/multi_asyncio.py,sha256=M3Jpx8Eak1XYiQRuO3MwBbZ58vZWJbsiju3ltb-XEPI,13002
|
24
|
-
jettask/monitoring/__init__.py,sha256=xWEEH5C2e8TW2iH6KNxBj5R-aTMg1GH3DrrwjRzRGD4,76
|
25
|
-
jettask/monitoring/file_watcher.py,sha256=r3Mgekb_5sOssDrnFBCbbyvpWkwD2ZPA_j22ztzRCT0,1207
|
26
|
-
jettask/utils/__init__.py,sha256=7oEoX1Au4GxQ_hbXjSc1q5titSBWsi-62UIIUbZ7zto,133
|
27
|
-
jettask/utils/error_handler.py,sha256=9iYHCInUkEkP9o07dUgPxQr4aWqR5FUhnNWLgB0TEqM,2794
|
28
|
-
jettask/utils/exception_hook.py,sha256=Fp0m71_jjmmZvyoELARSsPIvSVPCmMQMVC1FxMFOoHQ,1765
|
29
|
-
jettask/utils/helpers.py,sha256=uTdyRFZsdwSQzuEcNoGgADuVnHaDEYq6JQP6zVxAj04,3368
|
30
|
-
jettask/utils/logger.py,sha256=PcZgUtx7I9pe90Y8WgPQtKhMefd1oQ14pgUlKCFpCXo,843
|
31
|
-
jettask/utils/serializer.py,sha256=kTVDtPz1gwkmspeazNNjuCqrYxJLc2bzqmS8_I0Zxn0,714
|
32
|
-
jettask/utils/serializer_optimized.py,sha256=UijmvEX6WQBnaNL653z9YOmpxADdEtfS3wQMMBl5_KI,883
|
33
|
-
jettask/utils/task_logger.py,sha256=kauCgiDDzhuge3wk-BXR_KwytOfmrrVBKe1S4XUWBXU,13628
|
34
|
-
jettask/utils/traceback_filter.py,sha256=2svLo5_OoosAbS08UpceeWtjSKkdx-Fp6U-ysJQkroY,3025
|
35
|
-
jettask/webui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
36
|
-
jettask/webui/__main__.py,sha256=wzQJcE94Bz7uzb3x5GZ0qVt_zWt-IpdFMgEURGSgICM,5574
|
37
|
-
jettask/webui/api.py,sha256=rucIUVLfO3BfoSd12jX8BWIHTXpq9_uGDjGhudA_3JU,97400
|
38
|
-
jettask/webui/cleanup_deprecated_tables.sql,sha256=RmgwI1zx1qWiy4OXtTYPvBRFICkeqzC-1_xWOsnrrlI,529
|
39
|
-
jettask/webui/config.py,sha256=FOcbpmwZ0gGFSZbsE4l46l43Js2ER2-P9jEDTmptzgs,2619
|
40
|
-
jettask/webui/db_init.py,sha256=H2X4ifNgkgaApgu_TKnXNmfRzXvbv7AW6zpSwijawmY,7037
|
41
|
-
jettask/webui/gradio_app.py,sha256=ija3UDwWczbp5j0QwNxWw48MW-r9n4NsQkcxL67A14s,22650
|
42
|
-
jettask/webui/integrated_gradio_app.py,sha256=AmyrtlkVHxZ7yoHHPPQ0IAVebRgsmSQEUsCkPLbtFsI,44634
|
43
|
-
jettask/webui/models.py,sha256=QHYqU_6X5mTDFW_DA1aFQJ6kdmFRgVs4shqr7RJUXzk,2026
|
44
|
-
jettask/webui/pg_consumer.py,sha256=A5r3MYFHMcL5Ay4ryfTBWknVQaoc4dFzipisxhd5nUo,48863
|
45
|
-
jettask/webui/pg_consumer_slow.py,sha256=CQy5FAdmnzAB9lHxTfHUo2_j43uh7tyEganec0zigLA,49332
|
46
|
-
jettask/webui/pg_consumer_test.py,sha256=Ju8vomONsw1tLpY15KfUlXnKIYXAmXoSin0oZ2ub_QE,27575
|
47
|
-
jettask/webui/run.py,sha256=DwLhzPEBlkwO81-rs84vXZbFZDfVlEFv7_jT2nkbhco,1439
|
48
|
-
jettask/webui/run_monitor.py,sha256=n6aCsKE4BnjSTBPEJlFIU6b465pCcXJZZIsd5kUCvE0,562
|
49
|
-
jettask/webui/run_webui.py,sha256=BWIBtK3OB_K6S7mjiRGqGPSXSLo4irliZdZZQd66mYY,4519
|
50
|
-
jettask/webui/schema.sql,sha256=IY-vLzj3TYoR4p51ChLuSxzo4KU1PQNFNDutc7p0nHg,4490
|
51
|
-
jettask/webui/test_pg_consumer_recovery.py,sha256=lkN9RR0658X0um59TcKQj1EoN5cCmSmfFsJ0x07lqv0,21809
|
52
|
-
jettask/webui/test_recovery_simple.py,sha256=cqNkLYetWAdui71L6vHVHkomVb8bKN5LjgFlR_9BLOY,18417
|
53
|
-
jettask/webui/test_self_recovery.py,sha256=M6-9LhiXUDHV-fetKZRxwJFWe6S6ewB0fypu8jz8WpM,18192
|
54
|
-
jettask/webui/backend/__init__.py,sha256=OK2w-Nm0pF2nqNhhUoSRIDHFnAsQBJdPlIyp5_RU-Rw,32
|
55
|
-
jettask/webui/backend/data_access.py,sha256=TLiZ0YDjnstsmu8_KQcqA4UYI2OzuIVJU9Rm5q6RCCI,13812
|
56
|
-
jettask/webui/backend/main.py,sha256=H8wFstX_iA73VZ-EO4Y_DqDgZEgvMgxfDNI7BWVGbrY,6734
|
57
|
-
jettask/webui/backend/start.py,sha256=HMY7QMfWewmgJQTUamjzGV8YONTpjH4B9_W7fbL6_hE,1060
|
58
|
-
jettask/webui/frontend/package-lock.json,sha256=5hzk5tR06k8njepeAIGvy6rOBNuV7w5n6RFRvVADZ9w,162741
|
59
|
-
jettask/webui/frontend/package.json,sha256=4lCkSnaDwxzDr-jf-IcaqVCz7gEgMh7GRDb3NdZKbV0,615
|
60
|
-
jettask/webui/frontend/src/App.css,sha256=z4ISKU7bZ6unczb8b0cQtw9dG4KJ9iJvbZf0vDHsZtk,153
|
61
|
-
jettask/webui/frontend/src/App.jsx,sha256=QhwZTundKvKg7E6WJdVDwxQO1yzqeDOxpSntWaS2UqM,1210
|
62
|
-
jettask/webui/frontend/src/index.css,sha256=s2mxkzEDoY10XqRcV_5_32r4GZk22cvZoBMh6_NKVq0,2074
|
63
|
-
jettask/webui/frontend/src/main.jsx,sha256=Oht_gcLELsJ7oQQHvbOUyT_bobSf2HcSHgaXUG88ffc,467
|
64
|
-
jettask/webui/frontend/src/components/charts/QueueChart.jsx,sha256=UHtImnh_zRzMB6znfI8Sh8fIgZS4Py5a8Y2z88wI2fc,2796
|
65
|
-
jettask/webui/frontend/src/components/charts/QueueTrendChart.jsx,sha256=riJXsPFpgP_4FpB5Eea3DZVOpA_wpcgSQPnEhLe9oNo,2879
|
66
|
-
jettask/webui/frontend/src/components/charts/WorkerChart.jsx,sha256=jek5e97ioLlHcsEIBmoLsCi_j16X2eWgJo95g4sxQIo,901
|
67
|
-
jettask/webui/frontend/src/components/common/StatsCard.jsx,sha256=YYT2yZ9vPKq-GZE0NoTCjwvgyTlhoMulX6Zh-q9_j2s,360
|
68
|
-
jettask/webui/frontend/src/components/layout/Header.css,sha256=KGhnHcbumY0DheSc1jr2NCKUrP9NB7cLkTEWclww7AM,1379
|
69
|
-
jettask/webui/frontend/src/components/layout/Header.jsx,sha256=nqz5pMwZrDt2xBsLoEYIyE9AttVeHlgVWh3PzL0vkBY,2306
|
70
|
-
jettask/webui/frontend/src/contexts/LoadingContext.jsx,sha256=q-Ltc0teW0jt1RJrg2JdiaPiOZXmE9AenVeU8cRI8-M,715
|
71
|
-
jettask/webui/frontend/src/pages/QueueDetail.jsx,sha256=HlGHCZDe8x6FUb4DEJjRHI48TA5CK8ogi8kTbb8K3pI,491
|
72
|
-
jettask/webui/frontend/src/pages/QueueMonitor.jsx,sha256=LohX3nJWadVt2SQueZXbR5k_uXdfznn7rsNLvVvpGfo,11960
|
73
|
-
jettask/webui/frontend/src/pages/Queues.jsx,sha256=cD_2sWbdQZynZhc-lxLrMBqk71TphS_n6VwnPIhEzSg,143
|
74
|
-
jettask/webui/frontend/src/pages/Workers.jsx,sha256=S3JI-gDfkBuAIeHQdBCyFg8OuwOa970Kz4cgBGqaAmo,227
|
75
|
-
jettask/webui/frontend/src/pages/Dashboard/index.css,sha256=DNzBEusK0LtDgcVie_Tg-bcKHs_0EtLUPtqnBfYdERs,511
|
76
|
-
jettask/webui/frontend/src/pages/Dashboard/index.jsx,sha256=h-852VjLYcuoDGlkmJ0odrnLU554NsFh332lK99AzWI,7873
|
77
|
-
jettask/webui/frontend/src/services/api.js,sha256=IRXUIJ3fuFS4rX4cljfdbNpo62b2jwNyuvP7uMdRw9E,2807
|
78
|
-
jettask/webui/frontend/src/services/queueTrend.js,sha256=z-oADVYIUD923ilyAn0pVUzfhEmo6OGQ6RHRCvgfWgs,4591
|
79
|
-
jettask/webui/static/index.html,sha256=2XsZQyCmwmgCCHd20ugeRrGQb0jqI1p8YB4yRTsx6ZI,79389
|
80
|
-
jettask/webui/static/queue.html,sha256=UORFtntu0JQuUslMCxiuPaKxMl4dMzT1HgiUO1eYHno,46774
|
81
|
-
jettask/webui/static/queues.html,sha256=WkTaKSAydpwjemHMMVx_GTVyLf6Dz2Qy2Yu7UbMLf4E,20807
|
82
|
-
jettask/webui/static/workers.html,sha256=Tybg1uEcecAv6B5dCCK84deon9shRVBWGiTtqunkJkc,30015
|
83
|
-
jettask/webui/static/dist/index.html,sha256=jFbqhXkEi1GNJ1i6pK6E6puXD-Gy-RHT1Gi4Zmo9t04,454
|
84
|
-
jettask/webui/static/dist/assets/index-823408e8.css,sha256=gjQI6O_KKEynxMsC4MpBkToeF0lcHubBOWaxFoNiAOE,3235
|
85
|
-
jettask/webui/static/dist/assets/index-9968b0b8.js,sha256=d81auC3cAZWSE_xE0Z6kvxXAQ9XCMvMadMTlq82dIWM,2107979
|
86
|
-
jettask-0.2.1.dist-info/licenses/LICENSE,sha256=sKR8OPWvnqxzcHAmnaVSANpRpeM0Z52PNLCB0ZlFN6c,1062
|
87
|
-
jettask-0.2.1.dist-info/METADATA,sha256=iL4NR6pfmFvvR17dcu0_DTGibO4xNm028krEGDdVDJg,1858
|
88
|
-
jettask-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
89
|
-
jettask-0.2.1.dist-info/entry_points.txt,sha256=VC3byRkkSfRHu_QzczGtpGjcFERkJUlCrD__TFLVqxI,153
|
90
|
-
jettask-0.2.1.dist-info/top_level.txt,sha256=uymyRUF87-OsSurk5NhpeTW0jy3Wltnn91Zoa6jmAaw,8
|
91
|
-
jettask-0.2.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|