jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/constants.py +213 -0
- jettask/core/app.py +525 -205
- jettask/core/cli.py +193 -185
- jettask/core/consumer_manager.py +126 -34
- jettask/core/context.py +3 -0
- jettask/core/enums.py +137 -0
- jettask/core/event_pool.py +501 -168
- jettask/core/message.py +147 -0
- jettask/core/offline_worker_recovery.py +181 -114
- jettask/core/task.py +10 -174
- jettask/core/task_batch.py +153 -0
- jettask/core/unified_manager_base.py +243 -0
- jettask/core/worker_scanner.py +54 -54
- jettask/executors/asyncio.py +184 -64
- jettask/webui/backend/config.py +51 -0
- jettask/webui/backend/data_access.py +2083 -92
- jettask/webui/backend/data_api.py +3294 -0
- jettask/webui/backend/dependencies.py +261 -0
- jettask/webui/backend/init_meta_db.py +158 -0
- jettask/webui/backend/main.py +1358 -69
- jettask/webui/backend/main_unified.py +78 -0
- jettask/webui/backend/main_v2.py +394 -0
- jettask/webui/backend/namespace_api.py +295 -0
- jettask/webui/backend/namespace_api_old.py +294 -0
- jettask/webui/backend/namespace_data_access.py +611 -0
- jettask/webui/backend/queue_backlog_api.py +727 -0
- jettask/webui/backend/queue_stats_v2.py +521 -0
- jettask/webui/backend/redis_monitor_api.py +476 -0
- jettask/webui/backend/unified_api_router.py +1601 -0
- jettask/webui/db_init.py +204 -32
- jettask/webui/frontend/package-lock.json +492 -1
- jettask/webui/frontend/package.json +4 -1
- jettask/webui/frontend/src/App.css +105 -7
- jettask/webui/frontend/src/App.jsx +49 -20
- jettask/webui/frontend/src/components/NamespaceSelector.jsx +166 -0
- jettask/webui/frontend/src/components/QueueBacklogChart.jsx +298 -0
- jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +638 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.css +65 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.jsx +487 -0
- jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +465 -0
- jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +423 -0
- jettask/webui/frontend/src/components/TaskFilter.jsx +425 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.css +21 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.jsx +160 -0
- jettask/webui/frontend/src/components/layout/AppLayout.css +95 -0
- jettask/webui/frontend/src/components/layout/AppLayout.jsx +49 -0
- jettask/webui/frontend/src/components/layout/Header.css +34 -10
- jettask/webui/frontend/src/components/layout/Header.jsx +31 -23
- jettask/webui/frontend/src/components/layout/SideMenu.css +137 -0
- jettask/webui/frontend/src/components/layout/SideMenu.jsx +209 -0
- jettask/webui/frontend/src/components/layout/TabsNav.css +244 -0
- jettask/webui/frontend/src/components/layout/TabsNav.jsx +206 -0
- jettask/webui/frontend/src/components/layout/UserInfo.css +197 -0
- jettask/webui/frontend/src/components/layout/UserInfo.jsx +197 -0
- jettask/webui/frontend/src/contexts/NamespaceContext.jsx +72 -0
- jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +245 -0
- jettask/webui/frontend/src/main.jsx +1 -0
- jettask/webui/frontend/src/pages/Alerts.jsx +684 -0
- jettask/webui/frontend/src/pages/Dashboard.jsx +1330 -0
- jettask/webui/frontend/src/pages/QueueDetail.jsx +1109 -10
- jettask/webui/frontend/src/pages/QueueMonitor.jsx +236 -115
- jettask/webui/frontend/src/pages/Queues.jsx +5 -1
- jettask/webui/frontend/src/pages/ScheduledTasks.jsx +809 -0
- jettask/webui/frontend/src/pages/Settings.jsx +800 -0
- jettask/webui/frontend/src/services/api.js +7 -5
- jettask/webui/frontend/src/utils/suppressWarnings.js +22 -0
- jettask/webui/frontend/src/utils/userPreferences.js +154 -0
- jettask/webui/multi_namespace_consumer.py +543 -0
- jettask/webui/pg_consumer.py +983 -246
- jettask/webui/static/dist/assets/index-7129cfe1.css +1 -0
- jettask/webui/static/dist/assets/index-8d1935cc.js +774 -0
- jettask/webui/static/dist/index.html +2 -2
- jettask/webui/task_center.py +216 -0
- jettask/webui/task_center_client.py +150 -0
- jettask/webui/unified_consumer_manager.py +193 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/METADATA +1 -1
- jettask-0.2.4.dist-info/RECORD +134 -0
- jettask/webui/pg_consumer_slow.py +0 -1099
- jettask/webui/pg_consumer_test.py +0 -678
- jettask/webui/static/dist/assets/index-823408e8.css +0 -1
- jettask/webui/static/dist/assets/index-9968b0b8.js +0 -543
- jettask/webui/test_pg_consumer_recovery.py +0 -547
- jettask/webui/test_recovery_simple.py +0 -492
- jettask/webui/test_self_recovery.py +0 -467
- jettask-0.2.1.dist-info/RECORD +0 -91
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/WHEEL +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/top_level.txt +0 -0
@@ -1,678 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
"""测试版本的 PostgreSQL Consumer - 用于测试 pending 消息恢复"""
|
3
|
-
|
4
|
-
import asyncio
|
5
|
-
import json
|
6
|
-
import logging
|
7
|
-
import os
|
8
|
-
import time
|
9
|
-
import signal
|
10
|
-
from typing import Dict, List, Optional, Any
|
11
|
-
from datetime import datetime, timezone
|
12
|
-
from collections import defaultdict
|
13
|
-
|
14
|
-
import redis.asyncio as redis
|
15
|
-
from redis.asyncio import Redis
|
16
|
-
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
17
|
-
from sqlalchemy.orm import sessionmaker
|
18
|
-
from sqlalchemy import text
|
19
|
-
|
20
|
-
from jettask.webui.config import PostgreSQLConfig, RedisConfig
|
21
|
-
from jettask.core.consumer_manager import ConsumerManager, ConsumerStrategy
|
22
|
-
from jettask.core.offline_worker_recovery import OfflineWorkerRecovery
|
23
|
-
|
24
|
-
logger = logging.getLogger(__name__)
|
25
|
-
|
26
|
-
|
27
|
-
class TestPostgreSQLConsumer:
|
28
|
-
"""测试版本的PostgreSQL消费者,用于测试pending消息恢复"""
|
29
|
-
|
30
|
-
def __init__(self, pg_config: PostgreSQLConfig, redis_config: RedisConfig,
|
31
|
-
prefix: str = "jettask", node_id: str = None,
|
32
|
-
consumer_strategy: ConsumerStrategy = ConsumerStrategy.HEARTBEAT,
|
33
|
-
test_mode: str = None,
|
34
|
-
processing_delay: float = 0,
|
35
|
-
crash_after_messages: int = 0):
|
36
|
-
"""
|
37
|
-
Args:
|
38
|
-
test_mode: 测试模式
|
39
|
-
- "slow_process": 模拟缓慢处理
|
40
|
-
- "crash_after_n": 处理N条消息后崩溃
|
41
|
-
- "normal": 正常处理
|
42
|
-
processing_delay: 每条消息的处理延迟(秒)
|
43
|
-
crash_after_messages: 处理多少条消息后崩溃(仅在crash_after_n模式下有效)
|
44
|
-
"""
|
45
|
-
self.pg_config = pg_config
|
46
|
-
self.redis_config = redis_config
|
47
|
-
self.prefix = prefix
|
48
|
-
self.redis_client: Optional[Redis] = None
|
49
|
-
self.async_engine = None
|
50
|
-
self.AsyncSessionLocal = None
|
51
|
-
self.consumer_group = f"{prefix}_pg_consumer1"
|
52
|
-
|
53
|
-
# 测试配置
|
54
|
-
self.test_mode = test_mode or "normal"
|
55
|
-
self.processing_delay = processing_delay
|
56
|
-
self.crash_after_messages = crash_after_messages
|
57
|
-
self.processed_count = 0
|
58
|
-
self.queue_processed_count = defaultdict(int)
|
59
|
-
self.changes_processed_count = 0
|
60
|
-
|
61
|
-
# 节点标识
|
62
|
-
import socket
|
63
|
-
hostname = socket.gethostname()
|
64
|
-
self.node_id = node_id or f"{hostname}_{os.getpid()}"
|
65
|
-
|
66
|
-
# 使用 ConsumerManager 来管理 consumer_id
|
67
|
-
self.consumer_strategy = consumer_strategy
|
68
|
-
self.consumer_manager = None
|
69
|
-
self.consumer_id = None
|
70
|
-
|
71
|
-
self._running = False
|
72
|
-
self._tasks = []
|
73
|
-
self._known_queues = set()
|
74
|
-
self._consecutive_errors = defaultdict(int)
|
75
|
-
|
76
|
-
# 内存中维护已处理的任务ID集合
|
77
|
-
self._processed_task_ids = set()
|
78
|
-
self._processed_ids_lock = asyncio.Lock()
|
79
|
-
self._processed_ids_max_size = 100000
|
80
|
-
|
81
|
-
# 待重试的任务更新
|
82
|
-
self._pending_updates = {}
|
83
|
-
self._pending_updates_lock = asyncio.Lock()
|
84
|
-
self._max_pending_updates = 10000
|
85
|
-
self._retry_interval = 5
|
86
|
-
|
87
|
-
# 动态批次大小
|
88
|
-
self.batch_size = 100 # 减小批次便于测试
|
89
|
-
self.min_batch_size = 10
|
90
|
-
self.max_batch_size = 200
|
91
|
-
|
92
|
-
async def start(self):
|
93
|
-
"""启动消费者"""
|
94
|
-
logger.info(f"Starting TEST PostgreSQL consumer on node: {self.node_id}")
|
95
|
-
logger.info(f"Test mode: {self.test_mode}, delay: {self.processing_delay}s, crash after: {self.crash_after_messages}")
|
96
|
-
|
97
|
-
# 连接Redis
|
98
|
-
self.redis_client = await redis.Redis(
|
99
|
-
host=self.redis_config.host,
|
100
|
-
port=self.redis_config.port,
|
101
|
-
db=self.redis_config.db,
|
102
|
-
password=self.redis_config.password,
|
103
|
-
decode_responses=False
|
104
|
-
)
|
105
|
-
|
106
|
-
# 初始化 ConsumerManager
|
107
|
-
import redis as sync_redis
|
108
|
-
sync_redis_client = sync_redis.StrictRedis(
|
109
|
-
host=self.redis_config.host,
|
110
|
-
port=self.redis_config.port,
|
111
|
-
db=self.redis_config.db,
|
112
|
-
password=self.redis_config.password,
|
113
|
-
decode_responses=True
|
114
|
-
)
|
115
|
-
|
116
|
-
# 配置 ConsumerManager
|
117
|
-
initial_queues = ['TASK_CHANGES']
|
118
|
-
consumer_config = {
|
119
|
-
'redis_prefix': self.prefix,
|
120
|
-
'queues': initial_queues,
|
121
|
-
'worker_prefix': 'PG_CONSUMER',
|
122
|
-
}
|
123
|
-
|
124
|
-
self.consumer_manager = ConsumerManager(
|
125
|
-
redis_client=sync_redis_client,
|
126
|
-
strategy=self.consumer_strategy,
|
127
|
-
config=consumer_config
|
128
|
-
)
|
129
|
-
|
130
|
-
# 获取稳定的 consumer_id
|
131
|
-
self.consumer_id = self.consumer_manager.get_consumer_name('TASK_CHANGES')
|
132
|
-
logger.info(f"Using consumer_id: {self.consumer_id} with strategy: {self.consumer_strategy.value}")
|
133
|
-
|
134
|
-
# 创建SQLAlchemy异步引擎
|
135
|
-
if self.pg_config.dsn.startswith('postgresql://'):
|
136
|
-
dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+psycopg://', 1)
|
137
|
-
else:
|
138
|
-
dsn = self.pg_config.dsn
|
139
|
-
|
140
|
-
self.async_engine = create_async_engine(
|
141
|
-
dsn,
|
142
|
-
pool_size=10,
|
143
|
-
max_overflow=5,
|
144
|
-
pool_pre_ping=True,
|
145
|
-
pool_recycle=300,
|
146
|
-
echo=False
|
147
|
-
)
|
148
|
-
|
149
|
-
# 创建异步会话工厂
|
150
|
-
self.AsyncSessionLocal = sessionmaker(
|
151
|
-
self.async_engine,
|
152
|
-
class_=AsyncSession,
|
153
|
-
expire_on_commit=False
|
154
|
-
)
|
155
|
-
|
156
|
-
# 初始化数据库架构
|
157
|
-
await self._init_database()
|
158
|
-
|
159
|
-
self._running = True
|
160
|
-
|
161
|
-
# 先进行一次队列发现
|
162
|
-
await self._initial_queue_discovery()
|
163
|
-
|
164
|
-
# 创建离线worker恢复器
|
165
|
-
self.offline_recovery = OfflineWorkerRecovery(
|
166
|
-
async_redis_client=self.redis_client,
|
167
|
-
redis_prefix=self.prefix,
|
168
|
-
worker_prefix='PG_CONSUMER',
|
169
|
-
consumer_manager=self.consumer_manager
|
170
|
-
)
|
171
|
-
|
172
|
-
# 启动消费任务(简化版:只保留必要的任务)
|
173
|
-
self._tasks = [
|
174
|
-
asyncio.create_task(self._consume_queues()), # 消费新任务
|
175
|
-
asyncio.create_task(self._consume_task_changes()), # 消费任务变更事件
|
176
|
-
asyncio.create_task(self._start_offline_recovery()) # 离线worker恢复服务
|
177
|
-
]
|
178
|
-
|
179
|
-
logger.info("TEST PostgreSQL consumer started successfully")
|
180
|
-
|
181
|
-
async def stop(self):
|
182
|
-
"""停止消费者"""
|
183
|
-
logger.info("Stopping TEST PostgreSQL consumer...")
|
184
|
-
self._running = False
|
185
|
-
|
186
|
-
# 停止离线恢复服务
|
187
|
-
if hasattr(self, 'offline_recovery'):
|
188
|
-
self.offline_recovery.stop()
|
189
|
-
|
190
|
-
# 取消所有任务
|
191
|
-
for task in self._tasks:
|
192
|
-
task.cancel()
|
193
|
-
|
194
|
-
# 等待任务完成
|
195
|
-
await asyncio.gather(*self._tasks, return_exceptions=True)
|
196
|
-
|
197
|
-
# 清理 ConsumerManager
|
198
|
-
if self.consumer_manager:
|
199
|
-
try:
|
200
|
-
self.consumer_manager.cleanup()
|
201
|
-
logger.info(f"Cleaned up ConsumerManager for consumer: {self.consumer_id}")
|
202
|
-
except Exception as e:
|
203
|
-
logger.error(f"Error cleaning up ConsumerManager: {e}")
|
204
|
-
|
205
|
-
# 关闭连接
|
206
|
-
if self.redis_client:
|
207
|
-
await self.redis_client.close()
|
208
|
-
|
209
|
-
if self.async_engine:
|
210
|
-
await self.async_engine.dispose()
|
211
|
-
|
212
|
-
logger.info("TEST PostgreSQL consumer stopped")
|
213
|
-
|
214
|
-
async def _init_database(self):
|
215
|
-
"""初始化数据库架构"""
|
216
|
-
import os
|
217
|
-
current_dir = os.path.dirname(os.path.abspath(__file__))
|
218
|
-
schema_path = os.path.join(current_dir, "schema.sql")
|
219
|
-
try:
|
220
|
-
with open(schema_path, 'r') as f:
|
221
|
-
schema_sql = f.read()
|
222
|
-
|
223
|
-
async with self.AsyncSessionLocal() as session:
|
224
|
-
await session.execute(text(schema_sql))
|
225
|
-
await session.commit()
|
226
|
-
logger.info("Database schema initialized")
|
227
|
-
except FileNotFoundError:
|
228
|
-
logger.warning(f"Schema file not found at {schema_path}, skipping initialization")
|
229
|
-
except Exception as e:
|
230
|
-
logger.error(f"Failed to initialize database schema: {e}")
|
231
|
-
|
232
|
-
async def _initial_queue_discovery(self):
|
233
|
-
"""初始队列发现"""
|
234
|
-
try:
|
235
|
-
pattern = f"{self.prefix}:QUEUE:*"
|
236
|
-
new_queues = set()
|
237
|
-
|
238
|
-
async for key in self.redis_client.scan_iter(match=pattern, count=100):
|
239
|
-
queue_name = key.decode('utf-8').split(":")[-1]
|
240
|
-
new_queues.add(queue_name)
|
241
|
-
|
242
|
-
if new_queues:
|
243
|
-
all_queues = list(new_queues) + ['TASK_CHANGES']
|
244
|
-
|
245
|
-
if self.consumer_manager:
|
246
|
-
self.consumer_manager.config['queues'] = all_queues
|
247
|
-
|
248
|
-
if self.consumer_strategy == ConsumerStrategy.HEARTBEAT and hasattr(self.consumer_manager, '_heartbeat_strategy'):
|
249
|
-
actual_consumer_id = self.consumer_manager._heartbeat_strategy.consumer_id
|
250
|
-
else:
|
251
|
-
actual_consumer_id = self.consumer_id.rsplit('-', 1)[0] if '-' in self.consumer_id else self.consumer_id
|
252
|
-
|
253
|
-
worker_key = f"{self.prefix}:{self.consumer_manager.config.get('worker_prefix', 'PG_CONSUMER')}:{actual_consumer_id}"
|
254
|
-
try:
|
255
|
-
self.consumer_manager.redis_client.hset(
|
256
|
-
worker_key,
|
257
|
-
'queues',
|
258
|
-
','.join(all_queues)
|
259
|
-
)
|
260
|
-
logger.info(f"Initial queue discovery - found queues: {all_queues}")
|
261
|
-
except Exception as e:
|
262
|
-
logger.error(f"Error updating initial worker queues: {e}")
|
263
|
-
|
264
|
-
self._known_queues = new_queues
|
265
|
-
|
266
|
-
except Exception as e:
|
267
|
-
logger.error(f"Error in initial queue discovery: {e}")
|
268
|
-
|
269
|
-
async def _simulate_processing_delay(self, queue_name: str = None):
|
270
|
-
"""模拟处理延迟"""
|
271
|
-
if self.test_mode == "slow_process" and self.processing_delay > 0:
|
272
|
-
logger.info(f"[TEST] Simulating {self.processing_delay}s processing delay for {queue_name or 'message'}")
|
273
|
-
await asyncio.sleep(self.processing_delay)
|
274
|
-
|
275
|
-
async def _check_crash_condition(self, queue_name: str = None):
|
276
|
-
"""检查是否应该崩溃"""
|
277
|
-
if self.test_mode == "crash_after_n":
|
278
|
-
self.processed_count += 1
|
279
|
-
if queue_name:
|
280
|
-
self.queue_processed_count[queue_name] += 1
|
281
|
-
count = self.queue_processed_count[queue_name]
|
282
|
-
else:
|
283
|
-
self.changes_processed_count += 1
|
284
|
-
count = self.changes_processed_count
|
285
|
-
|
286
|
-
logger.info(f"[TEST] Processed {count} messages from {queue_name or 'TASK_CHANGES'}")
|
287
|
-
|
288
|
-
if self.crash_after_messages > 0 and count >= self.crash_after_messages:
|
289
|
-
logger.error(f"[TEST] CRASHING after processing {count} messages from {queue_name or 'TASK_CHANGES'}!")
|
290
|
-
# 强制退出进程
|
291
|
-
os._exit(1)
|
292
|
-
|
293
|
-
async def _consume_queue(self, queue_name: str):
|
294
|
-
"""消费单个队列的任务(带测试逻辑)"""
|
295
|
-
stream_key = f"{self.prefix}:QUEUE:{queue_name}"
|
296
|
-
check_backlog = True
|
297
|
-
lastid = "0-0"
|
298
|
-
|
299
|
-
consumer_name = self.consumer_id
|
300
|
-
|
301
|
-
while self._running and queue_name in self._known_queues:
|
302
|
-
try:
|
303
|
-
myid = lastid if check_backlog else ">"
|
304
|
-
|
305
|
-
# 减小批次大小便于测试
|
306
|
-
count = 50 if self.test_mode != "normal" else 10000
|
307
|
-
|
308
|
-
messages = await self.redis_client.xreadgroup(
|
309
|
-
self.consumer_group,
|
310
|
-
consumer_name,
|
311
|
-
{stream_key: myid},
|
312
|
-
count=count,
|
313
|
-
block=1000 if not check_backlog else 0
|
314
|
-
)
|
315
|
-
|
316
|
-
if not messages or (messages and len(messages[0][1]) == 0):
|
317
|
-
check_backlog = False
|
318
|
-
continue
|
319
|
-
|
320
|
-
if messages:
|
321
|
-
# 模拟处理延迟
|
322
|
-
await self._simulate_processing_delay(queue_name)
|
323
|
-
|
324
|
-
await self._process_messages(messages)
|
325
|
-
self._consecutive_errors[queue_name] = 0
|
326
|
-
|
327
|
-
# 检查崩溃条件
|
328
|
-
await self._check_crash_condition(queue_name)
|
329
|
-
|
330
|
-
if messages[0] and messages[0][1]:
|
331
|
-
lastid = messages[0][1][-1][0].decode('utf-8') if isinstance(messages[0][1][-1][0], bytes) else messages[0][1][-1][0]
|
332
|
-
check_backlog = len(messages[0][1]) >= count
|
333
|
-
|
334
|
-
except redis.ResponseError as e:
|
335
|
-
if "NOGROUP" in str(e):
|
336
|
-
try:
|
337
|
-
await self.redis_client.xgroup_create(
|
338
|
-
stream_key, self.consumer_group, id='0', mkstream=True
|
339
|
-
)
|
340
|
-
logger.info(f"Recreated consumer group for queue: {queue_name}")
|
341
|
-
check_backlog = True
|
342
|
-
lastid = "0-0"
|
343
|
-
except:
|
344
|
-
pass
|
345
|
-
else:
|
346
|
-
logger.error(f"Redis error for queue {queue_name}: {e}")
|
347
|
-
self._consecutive_errors[queue_name] += 1
|
348
|
-
|
349
|
-
if self._consecutive_errors[queue_name] > 10:
|
350
|
-
logger.warning(f"Too many errors for queue {queue_name}, will retry later")
|
351
|
-
await asyncio.sleep(30)
|
352
|
-
self._consecutive_errors[queue_name] = 0
|
353
|
-
|
354
|
-
except Exception as e:
|
355
|
-
logger.error(f"Error consuming queue {queue_name}: {e}", exc_info=True)
|
356
|
-
self._consecutive_errors[queue_name] += 1
|
357
|
-
await asyncio.sleep(1)
|
358
|
-
|
359
|
-
async def _consume_queues(self):
|
360
|
-
"""启动所有队列的消费任务"""
|
361
|
-
queue_tasks = {}
|
362
|
-
|
363
|
-
while self._running:
|
364
|
-
try:
|
365
|
-
for queue in self._known_queues:
|
366
|
-
if queue not in queue_tasks or queue_tasks[queue].done():
|
367
|
-
queue_tasks[queue] = asyncio.create_task(self._consume_queue(queue))
|
368
|
-
logger.info(f"Started consumer task for queue: {queue}")
|
369
|
-
|
370
|
-
for queue in list(queue_tasks.keys()):
|
371
|
-
if queue not in self._known_queues:
|
372
|
-
queue_tasks[queue].cancel()
|
373
|
-
del queue_tasks[queue]
|
374
|
-
logger.info(f"Stopped consumer task for removed queue: {queue}")
|
375
|
-
|
376
|
-
await asyncio.sleep(10)
|
377
|
-
|
378
|
-
except Exception as e:
|
379
|
-
logger.error(f"Error in consume_queues manager: {e}")
|
380
|
-
await asyncio.sleep(5)
|
381
|
-
|
382
|
-
for task in queue_tasks.values():
|
383
|
-
task.cancel()
|
384
|
-
|
385
|
-
await asyncio.gather(*queue_tasks.values(), return_exceptions=True)
|
386
|
-
|
387
|
-
async def _process_messages(self, messages: List):
|
388
|
-
"""处理消息并保存到PostgreSQL(简化版)"""
|
389
|
-
tasks_to_insert = []
|
390
|
-
ack_batch = []
|
391
|
-
|
392
|
-
for stream_key, stream_messages in messages:
|
393
|
-
if not stream_messages:
|
394
|
-
continue
|
395
|
-
|
396
|
-
stream_key_str = stream_key.decode('utf-8') if isinstance(stream_key, bytes) else stream_key
|
397
|
-
queue_name = stream_key_str.split(":")[-1]
|
398
|
-
msg_ids_to_ack = []
|
399
|
-
|
400
|
-
for msg_id, data in stream_messages:
|
401
|
-
try:
|
402
|
-
if not msg_id or not data:
|
403
|
-
continue
|
404
|
-
|
405
|
-
msg_id_str = msg_id.decode('utf-8') if isinstance(msg_id, bytes) else str(msg_id)
|
406
|
-
|
407
|
-
logger.info(f"[TEST] Processing message {msg_id_str} from queue {queue_name}")
|
408
|
-
|
409
|
-
# 简化处理,主要用于测试
|
410
|
-
task_info = {
|
411
|
-
'id': msg_id_str,
|
412
|
-
'queue_name': queue_name,
|
413
|
-
'task_name': 'test_task',
|
414
|
-
'task_data': '{}',
|
415
|
-
'priority': 0,
|
416
|
-
'retry_count': 0,
|
417
|
-
'max_retry': 3,
|
418
|
-
'status': 'pending',
|
419
|
-
'metadata': '{}',
|
420
|
-
'created_at': datetime.now(tz=timezone.utc)
|
421
|
-
}
|
422
|
-
tasks_to_insert.append(task_info)
|
423
|
-
msg_ids_to_ack.append(msg_id)
|
424
|
-
|
425
|
-
except Exception as e:
|
426
|
-
logger.error(f"Error processing message {msg_id}: {e}")
|
427
|
-
|
428
|
-
if msg_ids_to_ack:
|
429
|
-
ack_batch.append((stream_key, msg_ids_to_ack))
|
430
|
-
|
431
|
-
if tasks_to_insert:
|
432
|
-
# 简化:不真正插入数据库,只记录日志
|
433
|
-
logger.info(f"[TEST] Would insert {len(tasks_to_insert)} tasks to PostgreSQL")
|
434
|
-
|
435
|
-
if ack_batch:
|
436
|
-
pipeline = self.redis_client.pipeline()
|
437
|
-
for stream_key, msg_ids in ack_batch:
|
438
|
-
pipeline.xack(stream_key, self.consumer_group, *msg_ids)
|
439
|
-
|
440
|
-
try:
|
441
|
-
await pipeline.execute()
|
442
|
-
total_acked = sum(len(msg_ids) for _, msg_ids in ack_batch)
|
443
|
-
logger.info(f"[TEST] Successfully ACKed {total_acked} messages")
|
444
|
-
except Exception as e:
|
445
|
-
logger.error(f"Error executing batch ACK: {e}")
|
446
|
-
|
447
|
-
async def _consume_task_changes(self):
|
448
|
-
"""消费任务变更事件流(支持pending消息恢复)"""
|
449
|
-
change_stream_key = f"{self.prefix}:TASK_CHANGES"
|
450
|
-
consumer_group = f"{self.prefix}_changes_consumer"
|
451
|
-
|
452
|
-
consumer_name = self.consumer_manager.get_consumer_name('pg_consumer')
|
453
|
-
|
454
|
-
# 创建消费者组
|
455
|
-
try:
|
456
|
-
await self.redis_client.xgroup_create(
|
457
|
-
change_stream_key, consumer_group, id='0', mkstream=True
|
458
|
-
)
|
459
|
-
logger.info(f"Created consumer group for task changes stream")
|
460
|
-
except redis.ResponseError:
|
461
|
-
pass
|
462
|
-
|
463
|
-
# 模仿 listen_event_by_task 的写法
|
464
|
-
check_backlog = True
|
465
|
-
lastid = "0-0"
|
466
|
-
batch_size = 20 if self.test_mode != "normal" else 100
|
467
|
-
|
468
|
-
while self._running:
|
469
|
-
try:
|
470
|
-
if check_backlog:
|
471
|
-
myid = lastid
|
472
|
-
else:
|
473
|
-
myid = ">"
|
474
|
-
|
475
|
-
messages = await self.redis_client.xreadgroup(
|
476
|
-
consumer_group,
|
477
|
-
consumer_name,
|
478
|
-
{change_stream_key: myid},
|
479
|
-
count=batch_size,
|
480
|
-
block=1000 if not check_backlog else 0
|
481
|
-
)
|
482
|
-
|
483
|
-
if not messages:
|
484
|
-
check_backlog = False
|
485
|
-
continue
|
486
|
-
|
487
|
-
if messages and len(messages[0][1]) > 0:
|
488
|
-
check_backlog = len(messages[0][1]) >= batch_size
|
489
|
-
else:
|
490
|
-
check_backlog = False
|
491
|
-
|
492
|
-
task_ids_to_update = set()
|
493
|
-
ack_ids = []
|
494
|
-
|
495
|
-
for _, stream_messages in messages:
|
496
|
-
for msg_id, data in stream_messages:
|
497
|
-
try:
|
498
|
-
if isinstance(msg_id, bytes):
|
499
|
-
lastid = msg_id.decode('utf-8')
|
500
|
-
else:
|
501
|
-
lastid = str(msg_id)
|
502
|
-
|
503
|
-
event_id = data.get(b'event_id')
|
504
|
-
if event_id:
|
505
|
-
if isinstance(event_id, bytes):
|
506
|
-
event_id = event_id.decode('utf-8')
|
507
|
-
logger.info(f"[TEST] Processing TASK_CHANGES event: {event_id}")
|
508
|
-
task_ids_to_update.add(event_id)
|
509
|
-
ack_ids.append(msg_id)
|
510
|
-
except Exception as e:
|
511
|
-
logger.error(f"Error processing change event {msg_id}: {e}")
|
512
|
-
|
513
|
-
if task_ids_to_update:
|
514
|
-
# 模拟处理延迟
|
515
|
-
await self._simulate_processing_delay("TASK_CHANGES")
|
516
|
-
|
517
|
-
# 简化:不真正更新数据库
|
518
|
-
logger.info(f"[TEST] Would update {len(task_ids_to_update)} tasks from change events")
|
519
|
-
|
520
|
-
# 检查崩溃条件
|
521
|
-
await self._check_crash_condition("TASK_CHANGES")
|
522
|
-
|
523
|
-
if ack_ids:
|
524
|
-
await self.redis_client.xack(change_stream_key, consumer_group, *ack_ids)
|
525
|
-
logger.info(f"[TEST] ACKed {len(ack_ids)} TASK_CHANGES messages")
|
526
|
-
|
527
|
-
except redis.ResponseError as e:
|
528
|
-
if "NOGROUP" in str(e):
|
529
|
-
try:
|
530
|
-
await self.redis_client.xgroup_create(
|
531
|
-
change_stream_key, consumer_group, id='0', mkstream=True
|
532
|
-
)
|
533
|
-
logger.info(f"Recreated consumer group for task changes stream")
|
534
|
-
check_backlog = True
|
535
|
-
lastid = "0-0"
|
536
|
-
except:
|
537
|
-
pass
|
538
|
-
else:
|
539
|
-
logger.error(f"Redis error in consume_task_changes: {e}")
|
540
|
-
await asyncio.sleep(1)
|
541
|
-
except Exception as e:
|
542
|
-
logger.error(f"Error in consume_task_changes: {e}", exc_info=True)
|
543
|
-
await asyncio.sleep(1)
|
544
|
-
|
545
|
-
async def _start_offline_recovery(self):
|
546
|
-
"""启动离线worker恢复服务"""
|
547
|
-
logger.info("Starting offline worker recovery service for PG_CONSUMER")
|
548
|
-
|
549
|
-
while self._running:
|
550
|
-
try:
|
551
|
-
total_recovered = 0
|
552
|
-
|
553
|
-
# 恢复普通队列的消息
|
554
|
-
for queue in self._known_queues:
|
555
|
-
try:
|
556
|
-
recovered = await self.offline_recovery.recover_offline_workers(
|
557
|
-
queue=queue,
|
558
|
-
current_consumer_name=self.consumer_id,
|
559
|
-
process_message_callback=self._process_recovered_message
|
560
|
-
)
|
561
|
-
|
562
|
-
if recovered > 0:
|
563
|
-
logger.info(f"[TEST] Recovered {recovered} messages from queue {queue}")
|
564
|
-
total_recovered += recovered
|
565
|
-
|
566
|
-
except Exception as e:
|
567
|
-
logger.error(f"Error recovering queue {queue}: {e}")
|
568
|
-
|
569
|
-
if total_recovered > 0:
|
570
|
-
logger.info(f"[TEST] Total recovered {total_recovered} messages in this cycle")
|
571
|
-
|
572
|
-
# 减少扫描频率便于测试
|
573
|
-
await asyncio.sleep(5)
|
574
|
-
|
575
|
-
except Exception as e:
|
576
|
-
logger.error(f"Error in offline recovery service: {e}")
|
577
|
-
await asyncio.sleep(10)
|
578
|
-
|
579
|
-
async def _process_recovered_message(self, msg_id, msg_data, queue, consumer_id):
|
580
|
-
"""处理恢复的消息"""
|
581
|
-
try:
|
582
|
-
logger.info(f"[TEST] Processing recovered message {msg_id} from queue {queue}, offline worker {consumer_id}")
|
583
|
-
|
584
|
-
# ACK消息
|
585
|
-
if queue == 'TASK_CHANGES':
|
586
|
-
stream_key = f"{self.prefix}:TASK_CHANGES"
|
587
|
-
consumer_group = f"{self.prefix}_changes_consumer"
|
588
|
-
else:
|
589
|
-
stream_key = f"{self.prefix}:QUEUE:{queue}"
|
590
|
-
consumer_group = self.consumer_group
|
591
|
-
|
592
|
-
await self.redis_client.xack(stream_key, consumer_group, msg_id)
|
593
|
-
logger.info(f"[TEST] ACKed recovered message {msg_id}")
|
594
|
-
|
595
|
-
except Exception as e:
|
596
|
-
logger.error(f"Error processing recovered message {msg_id}: {e}")
|
597
|
-
|
598
|
-
|
599
|
-
async def run_test_pg_consumer(pg_config: PostgreSQLConfig, redis_config: RedisConfig,
|
600
|
-
test_mode: str = "normal",
|
601
|
-
processing_delay: float = 0,
|
602
|
-
crash_after_messages: int = 0,
|
603
|
-
node_id: str = None):
|
604
|
-
"""运行测试版PostgreSQL消费者"""
|
605
|
-
consumer = TestPostgreSQLConsumer(
|
606
|
-
pg_config,
|
607
|
-
redis_config,
|
608
|
-
node_id=node_id,
|
609
|
-
consumer_strategy=ConsumerStrategy.HEARTBEAT,
|
610
|
-
test_mode=test_mode,
|
611
|
-
processing_delay=processing_delay,
|
612
|
-
crash_after_messages=crash_after_messages
|
613
|
-
)
|
614
|
-
|
615
|
-
try:
|
616
|
-
await consumer.start()
|
617
|
-
while True:
|
618
|
-
await asyncio.sleep(1)
|
619
|
-
|
620
|
-
except KeyboardInterrupt:
|
621
|
-
logger.info("Received interrupt signal")
|
622
|
-
finally:
|
623
|
-
await consumer.stop()
|
624
|
-
|
625
|
-
|
626
|
-
def main():
|
627
|
-
"""主入口函数"""
|
628
|
-
import argparse
|
629
|
-
from dotenv import load_dotenv
|
630
|
-
|
631
|
-
load_dotenv()
|
632
|
-
|
633
|
-
parser = argparse.ArgumentParser(description='Test PostgreSQL Consumer')
|
634
|
-
parser.add_argument('--mode', choices=['normal', 'slow_process', 'crash_after_n'],
|
635
|
-
default='normal', help='Test mode')
|
636
|
-
parser.add_argument('--delay', type=float, default=0,
|
637
|
-
help='Processing delay in seconds')
|
638
|
-
parser.add_argument('--crash-after', type=int, default=0,
|
639
|
-
help='Crash after processing N messages')
|
640
|
-
parser.add_argument('--node-id', type=str, default=None,
|
641
|
-
help='Node ID for this consumer')
|
642
|
-
|
643
|
-
args = parser.parse_args()
|
644
|
-
|
645
|
-
logging.basicConfig(
|
646
|
-
level=logging.INFO,
|
647
|
-
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
648
|
-
)
|
649
|
-
|
650
|
-
pg_config = PostgreSQLConfig(
|
651
|
-
host=os.getenv('JETTASK_PG_HOST', 'localhost'),
|
652
|
-
port=int(os.getenv('JETTASK_PG_PORT', '5432')),
|
653
|
-
database=os.getenv('JETTASK_PG_DB', 'jettask'),
|
654
|
-
user=os.getenv('JETTASK_PG_USER', 'jettask'),
|
655
|
-
password=os.getenv('JETTASK_PG_PASSWORD', '123456'),
|
656
|
-
)
|
657
|
-
|
658
|
-
redis_config = RedisConfig(
|
659
|
-
host=os.getenv('REDIS_HOST', 'localhost'),
|
660
|
-
port=int(os.getenv('REDIS_PORT', '6379')),
|
661
|
-
db=int(os.getenv('REDIS_DB', '0')),
|
662
|
-
password=os.getenv('REDIS_PASSWORD'),
|
663
|
-
)
|
664
|
-
|
665
|
-
logger.info(f"Starting test consumer with mode={args.mode}, delay={args.delay}, crash_after={args.crash_after}")
|
666
|
-
|
667
|
-
asyncio.run(run_test_pg_consumer(
|
668
|
-
pg_config,
|
669
|
-
redis_config,
|
670
|
-
test_mode=args.mode,
|
671
|
-
processing_delay=args.delay,
|
672
|
-
crash_after_messages=args.crash_after,
|
673
|
-
node_id=args.node_id
|
674
|
-
))
|
675
|
-
|
676
|
-
|
677
|
-
if __name__ == '__main__':
|
678
|
-
main()
|
@@ -1 +0,0 @@
|
|
1
|
-
.app-header{background:rgba(20,20,20,.8)!important;-webkit-backdrop-filter:blur(20px);backdrop-filter:blur(20px);border-bottom:1px solid rgba(255,255,255,.08);padding:0;position:sticky;top:0;z-index:1000}.header-container{max-width:1440px;margin:0 auto;padding:0 24px;display:flex;align-items:center;justify-content:space-between;height:64px}.header-left{display:flex;align-items:center}.app-logo{display:flex;align-items:center;gap:12px;color:#fff;font-size:20px;font-weight:700;cursor:pointer}.logo-icon{font-size:24px;color:var(--primary-color)}.logo-text{background:linear-gradient(45deg,#1890ff,#52c41a);-webkit-background-clip:text;-webkit-text-fill-color:transparent;background-clip:text}.header-center{flex:1;display:flex;justify-content:center}.header-menu{background:transparent!important;border:none!important}.header-menu .ant-menu-item{color:#ffffffa6!important}.header-menu .ant-menu-item:hover{color:#fff!important}.header-menu .ant-menu-item-selected{color:var(--primary-color)!important}.header-right{display:flex;align-items:center}.refresh-btn{color:#ffffffa6!important}.refresh-btn:hover{color:#fff!important}.dashboard-container{padding:24px}.stats-row{margin-bottom:24px}.chart-controls{margin-bottom:16px}.connection-status{display:inline-flex;align-items:center;gap:8px;padding:4px 12px;border-radius:4px;font-size:12px}.connection-status.connected{background-color:#f6ffed;color:#52c41a}.connection-status.disconnected{background-color:#fff2e8;color:#fa8c16}.connection-status.error{background-color:#fff1f0;color:#ff4d4f}.app-layout{min-height:100vh;background:transparent}.main-content{padding:24px;max-width:1440px;margin:0 auto;width:100%}*{margin:0;padding:0;box-sizing:border-box}:root{--primary-color: #1890ff;--success-color: #52c41a;--warning-color: #faad14;--error-color: #ff4d4f;--bg-primary: #ffffff;--bg-secondary: #f5f5f5;--bg-card: #ffffff;--border-color: rgba(0, 0, 0, .06)}body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif;background:#f5f5f5;min-height:100vh;color:#000000d9}#root{position:relative;z-index:1;min-height:100vh}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:rgba(0,0,0,.05);border-radius:4px}::-webkit-scrollbar-thumb{background:rgba(0,0,0,.2);border-radius:4px}::-webkit-scrollbar-thumb:hover{background:rgba(0,0,0,.3)}.ant-layout{background:transparent!important}.ant-card{background:#ffffff!important;border:1px solid #f0f0f0}.stats-card{background:#ffffff;border-radius:8px;border:1px solid #f0f0f0;transition:all .3s cubic-bezier(.4,0,.2,1);overflow:hidden;position:relative}.stats-card:before{content:"";position:absolute;top:0;left:0;right:0;height:4px;background:linear-gradient(90deg,transparent,var(--primary-color),transparent);opacity:0;transition:opacity .3s}.stats-card:hover{transform:translateY(-4px);box-shadow:0 4px 12px #0000001a;border-color:#e8e8e8}.stats-card:hover:before{opacity:1}.chart-container{position:relative;height:100%;min-height:300px}.loading-spinner{display:inline-block;width:20px;height:20px;border:2px solid rgba(0,0,0,.1);border-top-color:var(--primary-color);border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}
|