jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. jettask/constants.py +213 -0
  2. jettask/core/app.py +525 -205
  3. jettask/core/cli.py +193 -185
  4. jettask/core/consumer_manager.py +126 -34
  5. jettask/core/context.py +3 -0
  6. jettask/core/enums.py +137 -0
  7. jettask/core/event_pool.py +501 -168
  8. jettask/core/message.py +147 -0
  9. jettask/core/offline_worker_recovery.py +181 -114
  10. jettask/core/task.py +10 -174
  11. jettask/core/task_batch.py +153 -0
  12. jettask/core/unified_manager_base.py +243 -0
  13. jettask/core/worker_scanner.py +54 -54
  14. jettask/executors/asyncio.py +184 -64
  15. jettask/webui/backend/config.py +51 -0
  16. jettask/webui/backend/data_access.py +2083 -92
  17. jettask/webui/backend/data_api.py +3294 -0
  18. jettask/webui/backend/dependencies.py +261 -0
  19. jettask/webui/backend/init_meta_db.py +158 -0
  20. jettask/webui/backend/main.py +1358 -69
  21. jettask/webui/backend/main_unified.py +78 -0
  22. jettask/webui/backend/main_v2.py +394 -0
  23. jettask/webui/backend/namespace_api.py +295 -0
  24. jettask/webui/backend/namespace_api_old.py +294 -0
  25. jettask/webui/backend/namespace_data_access.py +611 -0
  26. jettask/webui/backend/queue_backlog_api.py +727 -0
  27. jettask/webui/backend/queue_stats_v2.py +521 -0
  28. jettask/webui/backend/redis_monitor_api.py +476 -0
  29. jettask/webui/backend/unified_api_router.py +1601 -0
  30. jettask/webui/db_init.py +204 -32
  31. jettask/webui/frontend/package-lock.json +492 -1
  32. jettask/webui/frontend/package.json +4 -1
  33. jettask/webui/frontend/src/App.css +105 -7
  34. jettask/webui/frontend/src/App.jsx +49 -20
  35. jettask/webui/frontend/src/components/NamespaceSelector.jsx +166 -0
  36. jettask/webui/frontend/src/components/QueueBacklogChart.jsx +298 -0
  37. jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +638 -0
  38. jettask/webui/frontend/src/components/QueueDetailsTable.css +65 -0
  39. jettask/webui/frontend/src/components/QueueDetailsTable.jsx +487 -0
  40. jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +465 -0
  41. jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +423 -0
  42. jettask/webui/frontend/src/components/TaskFilter.jsx +425 -0
  43. jettask/webui/frontend/src/components/TimeRangeSelector.css +21 -0
  44. jettask/webui/frontend/src/components/TimeRangeSelector.jsx +160 -0
  45. jettask/webui/frontend/src/components/layout/AppLayout.css +95 -0
  46. jettask/webui/frontend/src/components/layout/AppLayout.jsx +49 -0
  47. jettask/webui/frontend/src/components/layout/Header.css +34 -10
  48. jettask/webui/frontend/src/components/layout/Header.jsx +31 -23
  49. jettask/webui/frontend/src/components/layout/SideMenu.css +137 -0
  50. jettask/webui/frontend/src/components/layout/SideMenu.jsx +209 -0
  51. jettask/webui/frontend/src/components/layout/TabsNav.css +244 -0
  52. jettask/webui/frontend/src/components/layout/TabsNav.jsx +206 -0
  53. jettask/webui/frontend/src/components/layout/UserInfo.css +197 -0
  54. jettask/webui/frontend/src/components/layout/UserInfo.jsx +197 -0
  55. jettask/webui/frontend/src/contexts/NamespaceContext.jsx +72 -0
  56. jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +245 -0
  57. jettask/webui/frontend/src/main.jsx +1 -0
  58. jettask/webui/frontend/src/pages/Alerts.jsx +684 -0
  59. jettask/webui/frontend/src/pages/Dashboard.jsx +1330 -0
  60. jettask/webui/frontend/src/pages/QueueDetail.jsx +1109 -10
  61. jettask/webui/frontend/src/pages/QueueMonitor.jsx +236 -115
  62. jettask/webui/frontend/src/pages/Queues.jsx +5 -1
  63. jettask/webui/frontend/src/pages/ScheduledTasks.jsx +809 -0
  64. jettask/webui/frontend/src/pages/Settings.jsx +800 -0
  65. jettask/webui/frontend/src/services/api.js +7 -5
  66. jettask/webui/frontend/src/utils/suppressWarnings.js +22 -0
  67. jettask/webui/frontend/src/utils/userPreferences.js +154 -0
  68. jettask/webui/multi_namespace_consumer.py +543 -0
  69. jettask/webui/pg_consumer.py +983 -246
  70. jettask/webui/static/dist/assets/index-7129cfe1.css +1 -0
  71. jettask/webui/static/dist/assets/index-8d1935cc.js +774 -0
  72. jettask/webui/static/dist/index.html +2 -2
  73. jettask/webui/task_center.py +216 -0
  74. jettask/webui/task_center_client.py +150 -0
  75. jettask/webui/unified_consumer_manager.py +193 -0
  76. {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/METADATA +1 -1
  77. jettask-0.2.4.dist-info/RECORD +134 -0
  78. jettask/webui/pg_consumer_slow.py +0 -1099
  79. jettask/webui/pg_consumer_test.py +0 -678
  80. jettask/webui/static/dist/assets/index-823408e8.css +0 -1
  81. jettask/webui/static/dist/assets/index-9968b0b8.js +0 -543
  82. jettask/webui/test_pg_consumer_recovery.py +0 -547
  83. jettask/webui/test_recovery_simple.py +0 -492
  84. jettask/webui/test_self_recovery.py +0 -467
  85. jettask-0.2.1.dist-info/RECORD +0 -91
  86. {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/WHEEL +0 -0
  87. {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/entry_points.txt +0 -0
  88. {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/licenses/LICENSE +0 -0
  89. {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/top_level.txt +0 -0
@@ -1,678 +0,0 @@
1
- #!/usr/bin/env python
2
- """测试版本的 PostgreSQL Consumer - 用于测试 pending 消息恢复"""
3
-
4
- import asyncio
5
- import json
6
- import logging
7
- import os
8
- import time
9
- import signal
10
- from typing import Dict, List, Optional, Any
11
- from datetime import datetime, timezone
12
- from collections import defaultdict
13
-
14
- import redis.asyncio as redis
15
- from redis.asyncio import Redis
16
- from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
17
- from sqlalchemy.orm import sessionmaker
18
- from sqlalchemy import text
19
-
20
- from jettask.webui.config import PostgreSQLConfig, RedisConfig
21
- from jettask.core.consumer_manager import ConsumerManager, ConsumerStrategy
22
- from jettask.core.offline_worker_recovery import OfflineWorkerRecovery
23
-
24
- logger = logging.getLogger(__name__)
25
-
26
-
27
- class TestPostgreSQLConsumer:
28
- """测试版本的PostgreSQL消费者,用于测试pending消息恢复"""
29
-
30
- def __init__(self, pg_config: PostgreSQLConfig, redis_config: RedisConfig,
31
- prefix: str = "jettask", node_id: str = None,
32
- consumer_strategy: ConsumerStrategy = ConsumerStrategy.HEARTBEAT,
33
- test_mode: str = None,
34
- processing_delay: float = 0,
35
- crash_after_messages: int = 0):
36
- """
37
- Args:
38
- test_mode: 测试模式
39
- - "slow_process": 模拟缓慢处理
40
- - "crash_after_n": 处理N条消息后崩溃
41
- - "normal": 正常处理
42
- processing_delay: 每条消息的处理延迟(秒)
43
- crash_after_messages: 处理多少条消息后崩溃(仅在crash_after_n模式下有效)
44
- """
45
- self.pg_config = pg_config
46
- self.redis_config = redis_config
47
- self.prefix = prefix
48
- self.redis_client: Optional[Redis] = None
49
- self.async_engine = None
50
- self.AsyncSessionLocal = None
51
- self.consumer_group = f"{prefix}_pg_consumer1"
52
-
53
- # 测试配置
54
- self.test_mode = test_mode or "normal"
55
- self.processing_delay = processing_delay
56
- self.crash_after_messages = crash_after_messages
57
- self.processed_count = 0
58
- self.queue_processed_count = defaultdict(int)
59
- self.changes_processed_count = 0
60
-
61
- # 节点标识
62
- import socket
63
- hostname = socket.gethostname()
64
- self.node_id = node_id or f"{hostname}_{os.getpid()}"
65
-
66
- # 使用 ConsumerManager 来管理 consumer_id
67
- self.consumer_strategy = consumer_strategy
68
- self.consumer_manager = None
69
- self.consumer_id = None
70
-
71
- self._running = False
72
- self._tasks = []
73
- self._known_queues = set()
74
- self._consecutive_errors = defaultdict(int)
75
-
76
- # 内存中维护已处理的任务ID集合
77
- self._processed_task_ids = set()
78
- self._processed_ids_lock = asyncio.Lock()
79
- self._processed_ids_max_size = 100000
80
-
81
- # 待重试的任务更新
82
- self._pending_updates = {}
83
- self._pending_updates_lock = asyncio.Lock()
84
- self._max_pending_updates = 10000
85
- self._retry_interval = 5
86
-
87
- # 动态批次大小
88
- self.batch_size = 100 # 减小批次便于测试
89
- self.min_batch_size = 10
90
- self.max_batch_size = 200
91
-
92
- async def start(self):
93
- """启动消费者"""
94
- logger.info(f"Starting TEST PostgreSQL consumer on node: {self.node_id}")
95
- logger.info(f"Test mode: {self.test_mode}, delay: {self.processing_delay}s, crash after: {self.crash_after_messages}")
96
-
97
- # 连接Redis
98
- self.redis_client = await redis.Redis(
99
- host=self.redis_config.host,
100
- port=self.redis_config.port,
101
- db=self.redis_config.db,
102
- password=self.redis_config.password,
103
- decode_responses=False
104
- )
105
-
106
- # 初始化 ConsumerManager
107
- import redis as sync_redis
108
- sync_redis_client = sync_redis.StrictRedis(
109
- host=self.redis_config.host,
110
- port=self.redis_config.port,
111
- db=self.redis_config.db,
112
- password=self.redis_config.password,
113
- decode_responses=True
114
- )
115
-
116
- # 配置 ConsumerManager
117
- initial_queues = ['TASK_CHANGES']
118
- consumer_config = {
119
- 'redis_prefix': self.prefix,
120
- 'queues': initial_queues,
121
- 'worker_prefix': 'PG_CONSUMER',
122
- }
123
-
124
- self.consumer_manager = ConsumerManager(
125
- redis_client=sync_redis_client,
126
- strategy=self.consumer_strategy,
127
- config=consumer_config
128
- )
129
-
130
- # 获取稳定的 consumer_id
131
- self.consumer_id = self.consumer_manager.get_consumer_name('TASK_CHANGES')
132
- logger.info(f"Using consumer_id: {self.consumer_id} with strategy: {self.consumer_strategy.value}")
133
-
134
- # 创建SQLAlchemy异步引擎
135
- if self.pg_config.dsn.startswith('postgresql://'):
136
- dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+psycopg://', 1)
137
- else:
138
- dsn = self.pg_config.dsn
139
-
140
- self.async_engine = create_async_engine(
141
- dsn,
142
- pool_size=10,
143
- max_overflow=5,
144
- pool_pre_ping=True,
145
- pool_recycle=300,
146
- echo=False
147
- )
148
-
149
- # 创建异步会话工厂
150
- self.AsyncSessionLocal = sessionmaker(
151
- self.async_engine,
152
- class_=AsyncSession,
153
- expire_on_commit=False
154
- )
155
-
156
- # 初始化数据库架构
157
- await self._init_database()
158
-
159
- self._running = True
160
-
161
- # 先进行一次队列发现
162
- await self._initial_queue_discovery()
163
-
164
- # 创建离线worker恢复器
165
- self.offline_recovery = OfflineWorkerRecovery(
166
- async_redis_client=self.redis_client,
167
- redis_prefix=self.prefix,
168
- worker_prefix='PG_CONSUMER',
169
- consumer_manager=self.consumer_manager
170
- )
171
-
172
- # 启动消费任务(简化版:只保留必要的任务)
173
- self._tasks = [
174
- asyncio.create_task(self._consume_queues()), # 消费新任务
175
- asyncio.create_task(self._consume_task_changes()), # 消费任务变更事件
176
- asyncio.create_task(self._start_offline_recovery()) # 离线worker恢复服务
177
- ]
178
-
179
- logger.info("TEST PostgreSQL consumer started successfully")
180
-
181
- async def stop(self):
182
- """停止消费者"""
183
- logger.info("Stopping TEST PostgreSQL consumer...")
184
- self._running = False
185
-
186
- # 停止离线恢复服务
187
- if hasattr(self, 'offline_recovery'):
188
- self.offline_recovery.stop()
189
-
190
- # 取消所有任务
191
- for task in self._tasks:
192
- task.cancel()
193
-
194
- # 等待任务完成
195
- await asyncio.gather(*self._tasks, return_exceptions=True)
196
-
197
- # 清理 ConsumerManager
198
- if self.consumer_manager:
199
- try:
200
- self.consumer_manager.cleanup()
201
- logger.info(f"Cleaned up ConsumerManager for consumer: {self.consumer_id}")
202
- except Exception as e:
203
- logger.error(f"Error cleaning up ConsumerManager: {e}")
204
-
205
- # 关闭连接
206
- if self.redis_client:
207
- await self.redis_client.close()
208
-
209
- if self.async_engine:
210
- await self.async_engine.dispose()
211
-
212
- logger.info("TEST PostgreSQL consumer stopped")
213
-
214
- async def _init_database(self):
215
- """初始化数据库架构"""
216
- import os
217
- current_dir = os.path.dirname(os.path.abspath(__file__))
218
- schema_path = os.path.join(current_dir, "schema.sql")
219
- try:
220
- with open(schema_path, 'r') as f:
221
- schema_sql = f.read()
222
-
223
- async with self.AsyncSessionLocal() as session:
224
- await session.execute(text(schema_sql))
225
- await session.commit()
226
- logger.info("Database schema initialized")
227
- except FileNotFoundError:
228
- logger.warning(f"Schema file not found at {schema_path}, skipping initialization")
229
- except Exception as e:
230
- logger.error(f"Failed to initialize database schema: {e}")
231
-
232
- async def _initial_queue_discovery(self):
233
- """初始队列发现"""
234
- try:
235
- pattern = f"{self.prefix}:QUEUE:*"
236
- new_queues = set()
237
-
238
- async for key in self.redis_client.scan_iter(match=pattern, count=100):
239
- queue_name = key.decode('utf-8').split(":")[-1]
240
- new_queues.add(queue_name)
241
-
242
- if new_queues:
243
- all_queues = list(new_queues) + ['TASK_CHANGES']
244
-
245
- if self.consumer_manager:
246
- self.consumer_manager.config['queues'] = all_queues
247
-
248
- if self.consumer_strategy == ConsumerStrategy.HEARTBEAT and hasattr(self.consumer_manager, '_heartbeat_strategy'):
249
- actual_consumer_id = self.consumer_manager._heartbeat_strategy.consumer_id
250
- else:
251
- actual_consumer_id = self.consumer_id.rsplit('-', 1)[0] if '-' in self.consumer_id else self.consumer_id
252
-
253
- worker_key = f"{self.prefix}:{self.consumer_manager.config.get('worker_prefix', 'PG_CONSUMER')}:{actual_consumer_id}"
254
- try:
255
- self.consumer_manager.redis_client.hset(
256
- worker_key,
257
- 'queues',
258
- ','.join(all_queues)
259
- )
260
- logger.info(f"Initial queue discovery - found queues: {all_queues}")
261
- except Exception as e:
262
- logger.error(f"Error updating initial worker queues: {e}")
263
-
264
- self._known_queues = new_queues
265
-
266
- except Exception as e:
267
- logger.error(f"Error in initial queue discovery: {e}")
268
-
269
- async def _simulate_processing_delay(self, queue_name: str = None):
270
- """模拟处理延迟"""
271
- if self.test_mode == "slow_process" and self.processing_delay > 0:
272
- logger.info(f"[TEST] Simulating {self.processing_delay}s processing delay for {queue_name or 'message'}")
273
- await asyncio.sleep(self.processing_delay)
274
-
275
- async def _check_crash_condition(self, queue_name: str = None):
276
- """检查是否应该崩溃"""
277
- if self.test_mode == "crash_after_n":
278
- self.processed_count += 1
279
- if queue_name:
280
- self.queue_processed_count[queue_name] += 1
281
- count = self.queue_processed_count[queue_name]
282
- else:
283
- self.changes_processed_count += 1
284
- count = self.changes_processed_count
285
-
286
- logger.info(f"[TEST] Processed {count} messages from {queue_name or 'TASK_CHANGES'}")
287
-
288
- if self.crash_after_messages > 0 and count >= self.crash_after_messages:
289
- logger.error(f"[TEST] CRASHING after processing {count} messages from {queue_name or 'TASK_CHANGES'}!")
290
- # 强制退出进程
291
- os._exit(1)
292
-
293
- async def _consume_queue(self, queue_name: str):
294
- """消费单个队列的任务(带测试逻辑)"""
295
- stream_key = f"{self.prefix}:QUEUE:{queue_name}"
296
- check_backlog = True
297
- lastid = "0-0"
298
-
299
- consumer_name = self.consumer_id
300
-
301
- while self._running and queue_name in self._known_queues:
302
- try:
303
- myid = lastid if check_backlog else ">"
304
-
305
- # 减小批次大小便于测试
306
- count = 50 if self.test_mode != "normal" else 10000
307
-
308
- messages = await self.redis_client.xreadgroup(
309
- self.consumer_group,
310
- consumer_name,
311
- {stream_key: myid},
312
- count=count,
313
- block=1000 if not check_backlog else 0
314
- )
315
-
316
- if not messages or (messages and len(messages[0][1]) == 0):
317
- check_backlog = False
318
- continue
319
-
320
- if messages:
321
- # 模拟处理延迟
322
- await self._simulate_processing_delay(queue_name)
323
-
324
- await self._process_messages(messages)
325
- self._consecutive_errors[queue_name] = 0
326
-
327
- # 检查崩溃条件
328
- await self._check_crash_condition(queue_name)
329
-
330
- if messages[0] and messages[0][1]:
331
- lastid = messages[0][1][-1][0].decode('utf-8') if isinstance(messages[0][1][-1][0], bytes) else messages[0][1][-1][0]
332
- check_backlog = len(messages[0][1]) >= count
333
-
334
- except redis.ResponseError as e:
335
- if "NOGROUP" in str(e):
336
- try:
337
- await self.redis_client.xgroup_create(
338
- stream_key, self.consumer_group, id='0', mkstream=True
339
- )
340
- logger.info(f"Recreated consumer group for queue: {queue_name}")
341
- check_backlog = True
342
- lastid = "0-0"
343
- except:
344
- pass
345
- else:
346
- logger.error(f"Redis error for queue {queue_name}: {e}")
347
- self._consecutive_errors[queue_name] += 1
348
-
349
- if self._consecutive_errors[queue_name] > 10:
350
- logger.warning(f"Too many errors for queue {queue_name}, will retry later")
351
- await asyncio.sleep(30)
352
- self._consecutive_errors[queue_name] = 0
353
-
354
- except Exception as e:
355
- logger.error(f"Error consuming queue {queue_name}: {e}", exc_info=True)
356
- self._consecutive_errors[queue_name] += 1
357
- await asyncio.sleep(1)
358
-
359
- async def _consume_queues(self):
360
- """启动所有队列的消费任务"""
361
- queue_tasks = {}
362
-
363
- while self._running:
364
- try:
365
- for queue in self._known_queues:
366
- if queue not in queue_tasks or queue_tasks[queue].done():
367
- queue_tasks[queue] = asyncio.create_task(self._consume_queue(queue))
368
- logger.info(f"Started consumer task for queue: {queue}")
369
-
370
- for queue in list(queue_tasks.keys()):
371
- if queue not in self._known_queues:
372
- queue_tasks[queue].cancel()
373
- del queue_tasks[queue]
374
- logger.info(f"Stopped consumer task for removed queue: {queue}")
375
-
376
- await asyncio.sleep(10)
377
-
378
- except Exception as e:
379
- logger.error(f"Error in consume_queues manager: {e}")
380
- await asyncio.sleep(5)
381
-
382
- for task in queue_tasks.values():
383
- task.cancel()
384
-
385
- await asyncio.gather(*queue_tasks.values(), return_exceptions=True)
386
-
387
- async def _process_messages(self, messages: List):
388
- """处理消息并保存到PostgreSQL(简化版)"""
389
- tasks_to_insert = []
390
- ack_batch = []
391
-
392
- for stream_key, stream_messages in messages:
393
- if not stream_messages:
394
- continue
395
-
396
- stream_key_str = stream_key.decode('utf-8') if isinstance(stream_key, bytes) else stream_key
397
- queue_name = stream_key_str.split(":")[-1]
398
- msg_ids_to_ack = []
399
-
400
- for msg_id, data in stream_messages:
401
- try:
402
- if not msg_id or not data:
403
- continue
404
-
405
- msg_id_str = msg_id.decode('utf-8') if isinstance(msg_id, bytes) else str(msg_id)
406
-
407
- logger.info(f"[TEST] Processing message {msg_id_str} from queue {queue_name}")
408
-
409
- # 简化处理,主要用于测试
410
- task_info = {
411
- 'id': msg_id_str,
412
- 'queue_name': queue_name,
413
- 'task_name': 'test_task',
414
- 'task_data': '{}',
415
- 'priority': 0,
416
- 'retry_count': 0,
417
- 'max_retry': 3,
418
- 'status': 'pending',
419
- 'metadata': '{}',
420
- 'created_at': datetime.now(tz=timezone.utc)
421
- }
422
- tasks_to_insert.append(task_info)
423
- msg_ids_to_ack.append(msg_id)
424
-
425
- except Exception as e:
426
- logger.error(f"Error processing message {msg_id}: {e}")
427
-
428
- if msg_ids_to_ack:
429
- ack_batch.append((stream_key, msg_ids_to_ack))
430
-
431
- if tasks_to_insert:
432
- # 简化:不真正插入数据库,只记录日志
433
- logger.info(f"[TEST] Would insert {len(tasks_to_insert)} tasks to PostgreSQL")
434
-
435
- if ack_batch:
436
- pipeline = self.redis_client.pipeline()
437
- for stream_key, msg_ids in ack_batch:
438
- pipeline.xack(stream_key, self.consumer_group, *msg_ids)
439
-
440
- try:
441
- await pipeline.execute()
442
- total_acked = sum(len(msg_ids) for _, msg_ids in ack_batch)
443
- logger.info(f"[TEST] Successfully ACKed {total_acked} messages")
444
- except Exception as e:
445
- logger.error(f"Error executing batch ACK: {e}")
446
-
447
- async def _consume_task_changes(self):
448
- """消费任务变更事件流(支持pending消息恢复)"""
449
- change_stream_key = f"{self.prefix}:TASK_CHANGES"
450
- consumer_group = f"{self.prefix}_changes_consumer"
451
-
452
- consumer_name = self.consumer_manager.get_consumer_name('pg_consumer')
453
-
454
- # 创建消费者组
455
- try:
456
- await self.redis_client.xgroup_create(
457
- change_stream_key, consumer_group, id='0', mkstream=True
458
- )
459
- logger.info(f"Created consumer group for task changes stream")
460
- except redis.ResponseError:
461
- pass
462
-
463
- # 模仿 listen_event_by_task 的写法
464
- check_backlog = True
465
- lastid = "0-0"
466
- batch_size = 20 if self.test_mode != "normal" else 100
467
-
468
- while self._running:
469
- try:
470
- if check_backlog:
471
- myid = lastid
472
- else:
473
- myid = ">"
474
-
475
- messages = await self.redis_client.xreadgroup(
476
- consumer_group,
477
- consumer_name,
478
- {change_stream_key: myid},
479
- count=batch_size,
480
- block=1000 if not check_backlog else 0
481
- )
482
-
483
- if not messages:
484
- check_backlog = False
485
- continue
486
-
487
- if messages and len(messages[0][1]) > 0:
488
- check_backlog = len(messages[0][1]) >= batch_size
489
- else:
490
- check_backlog = False
491
-
492
- task_ids_to_update = set()
493
- ack_ids = []
494
-
495
- for _, stream_messages in messages:
496
- for msg_id, data in stream_messages:
497
- try:
498
- if isinstance(msg_id, bytes):
499
- lastid = msg_id.decode('utf-8')
500
- else:
501
- lastid = str(msg_id)
502
-
503
- event_id = data.get(b'event_id')
504
- if event_id:
505
- if isinstance(event_id, bytes):
506
- event_id = event_id.decode('utf-8')
507
- logger.info(f"[TEST] Processing TASK_CHANGES event: {event_id}")
508
- task_ids_to_update.add(event_id)
509
- ack_ids.append(msg_id)
510
- except Exception as e:
511
- logger.error(f"Error processing change event {msg_id}: {e}")
512
-
513
- if task_ids_to_update:
514
- # 模拟处理延迟
515
- await self._simulate_processing_delay("TASK_CHANGES")
516
-
517
- # 简化:不真正更新数据库
518
- logger.info(f"[TEST] Would update {len(task_ids_to_update)} tasks from change events")
519
-
520
- # 检查崩溃条件
521
- await self._check_crash_condition("TASK_CHANGES")
522
-
523
- if ack_ids:
524
- await self.redis_client.xack(change_stream_key, consumer_group, *ack_ids)
525
- logger.info(f"[TEST] ACKed {len(ack_ids)} TASK_CHANGES messages")
526
-
527
- except redis.ResponseError as e:
528
- if "NOGROUP" in str(e):
529
- try:
530
- await self.redis_client.xgroup_create(
531
- change_stream_key, consumer_group, id='0', mkstream=True
532
- )
533
- logger.info(f"Recreated consumer group for task changes stream")
534
- check_backlog = True
535
- lastid = "0-0"
536
- except:
537
- pass
538
- else:
539
- logger.error(f"Redis error in consume_task_changes: {e}")
540
- await asyncio.sleep(1)
541
- except Exception as e:
542
- logger.error(f"Error in consume_task_changes: {e}", exc_info=True)
543
- await asyncio.sleep(1)
544
-
545
- async def _start_offline_recovery(self):
546
- """启动离线worker恢复服务"""
547
- logger.info("Starting offline worker recovery service for PG_CONSUMER")
548
-
549
- while self._running:
550
- try:
551
- total_recovered = 0
552
-
553
- # 恢复普通队列的消息
554
- for queue in self._known_queues:
555
- try:
556
- recovered = await self.offline_recovery.recover_offline_workers(
557
- queue=queue,
558
- current_consumer_name=self.consumer_id,
559
- process_message_callback=self._process_recovered_message
560
- )
561
-
562
- if recovered > 0:
563
- logger.info(f"[TEST] Recovered {recovered} messages from queue {queue}")
564
- total_recovered += recovered
565
-
566
- except Exception as e:
567
- logger.error(f"Error recovering queue {queue}: {e}")
568
-
569
- if total_recovered > 0:
570
- logger.info(f"[TEST] Total recovered {total_recovered} messages in this cycle")
571
-
572
- # 减少扫描频率便于测试
573
- await asyncio.sleep(5)
574
-
575
- except Exception as e:
576
- logger.error(f"Error in offline recovery service: {e}")
577
- await asyncio.sleep(10)
578
-
579
- async def _process_recovered_message(self, msg_id, msg_data, queue, consumer_id):
580
- """处理恢复的消息"""
581
- try:
582
- logger.info(f"[TEST] Processing recovered message {msg_id} from queue {queue}, offline worker {consumer_id}")
583
-
584
- # ACK消息
585
- if queue == 'TASK_CHANGES':
586
- stream_key = f"{self.prefix}:TASK_CHANGES"
587
- consumer_group = f"{self.prefix}_changes_consumer"
588
- else:
589
- stream_key = f"{self.prefix}:QUEUE:{queue}"
590
- consumer_group = self.consumer_group
591
-
592
- await self.redis_client.xack(stream_key, consumer_group, msg_id)
593
- logger.info(f"[TEST] ACKed recovered message {msg_id}")
594
-
595
- except Exception as e:
596
- logger.error(f"Error processing recovered message {msg_id}: {e}")
597
-
598
-
599
- async def run_test_pg_consumer(pg_config: PostgreSQLConfig, redis_config: RedisConfig,
600
- test_mode: str = "normal",
601
- processing_delay: float = 0,
602
- crash_after_messages: int = 0,
603
- node_id: str = None):
604
- """运行测试版PostgreSQL消费者"""
605
- consumer = TestPostgreSQLConsumer(
606
- pg_config,
607
- redis_config,
608
- node_id=node_id,
609
- consumer_strategy=ConsumerStrategy.HEARTBEAT,
610
- test_mode=test_mode,
611
- processing_delay=processing_delay,
612
- crash_after_messages=crash_after_messages
613
- )
614
-
615
- try:
616
- await consumer.start()
617
- while True:
618
- await asyncio.sleep(1)
619
-
620
- except KeyboardInterrupt:
621
- logger.info("Received interrupt signal")
622
- finally:
623
- await consumer.stop()
624
-
625
-
626
- def main():
627
- """主入口函数"""
628
- import argparse
629
- from dotenv import load_dotenv
630
-
631
- load_dotenv()
632
-
633
- parser = argparse.ArgumentParser(description='Test PostgreSQL Consumer')
634
- parser.add_argument('--mode', choices=['normal', 'slow_process', 'crash_after_n'],
635
- default='normal', help='Test mode')
636
- parser.add_argument('--delay', type=float, default=0,
637
- help='Processing delay in seconds')
638
- parser.add_argument('--crash-after', type=int, default=0,
639
- help='Crash after processing N messages')
640
- parser.add_argument('--node-id', type=str, default=None,
641
- help='Node ID for this consumer')
642
-
643
- args = parser.parse_args()
644
-
645
- logging.basicConfig(
646
- level=logging.INFO,
647
- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
648
- )
649
-
650
- pg_config = PostgreSQLConfig(
651
- host=os.getenv('JETTASK_PG_HOST', 'localhost'),
652
- port=int(os.getenv('JETTASK_PG_PORT', '5432')),
653
- database=os.getenv('JETTASK_PG_DB', 'jettask'),
654
- user=os.getenv('JETTASK_PG_USER', 'jettask'),
655
- password=os.getenv('JETTASK_PG_PASSWORD', '123456'),
656
- )
657
-
658
- redis_config = RedisConfig(
659
- host=os.getenv('REDIS_HOST', 'localhost'),
660
- port=int(os.getenv('REDIS_PORT', '6379')),
661
- db=int(os.getenv('REDIS_DB', '0')),
662
- password=os.getenv('REDIS_PASSWORD'),
663
- )
664
-
665
- logger.info(f"Starting test consumer with mode={args.mode}, delay={args.delay}, crash_after={args.crash_after}")
666
-
667
- asyncio.run(run_test_pg_consumer(
668
- pg_config,
669
- redis_config,
670
- test_mode=args.mode,
671
- processing_delay=args.delay,
672
- crash_after_messages=args.crash_after,
673
- node_id=args.node_id
674
- ))
675
-
676
-
677
- if __name__ == '__main__':
678
- main()
@@ -1 +0,0 @@
1
- .app-header{background:rgba(20,20,20,.8)!important;-webkit-backdrop-filter:blur(20px);backdrop-filter:blur(20px);border-bottom:1px solid rgba(255,255,255,.08);padding:0;position:sticky;top:0;z-index:1000}.header-container{max-width:1440px;margin:0 auto;padding:0 24px;display:flex;align-items:center;justify-content:space-between;height:64px}.header-left{display:flex;align-items:center}.app-logo{display:flex;align-items:center;gap:12px;color:#fff;font-size:20px;font-weight:700;cursor:pointer}.logo-icon{font-size:24px;color:var(--primary-color)}.logo-text{background:linear-gradient(45deg,#1890ff,#52c41a);-webkit-background-clip:text;-webkit-text-fill-color:transparent;background-clip:text}.header-center{flex:1;display:flex;justify-content:center}.header-menu{background:transparent!important;border:none!important}.header-menu .ant-menu-item{color:#ffffffa6!important}.header-menu .ant-menu-item:hover{color:#fff!important}.header-menu .ant-menu-item-selected{color:var(--primary-color)!important}.header-right{display:flex;align-items:center}.refresh-btn{color:#ffffffa6!important}.refresh-btn:hover{color:#fff!important}.dashboard-container{padding:24px}.stats-row{margin-bottom:24px}.chart-controls{margin-bottom:16px}.connection-status{display:inline-flex;align-items:center;gap:8px;padding:4px 12px;border-radius:4px;font-size:12px}.connection-status.connected{background-color:#f6ffed;color:#52c41a}.connection-status.disconnected{background-color:#fff2e8;color:#fa8c16}.connection-status.error{background-color:#fff1f0;color:#ff4d4f}.app-layout{min-height:100vh;background:transparent}.main-content{padding:24px;max-width:1440px;margin:0 auto;width:100%}*{margin:0;padding:0;box-sizing:border-box}:root{--primary-color: #1890ff;--success-color: #52c41a;--warning-color: #faad14;--error-color: #ff4d4f;--bg-primary: #ffffff;--bg-secondary: #f5f5f5;--bg-card: #ffffff;--border-color: rgba(0, 0, 0, .06)}body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,sans-serif;background:#f5f5f5;min-height:100vh;color:#000000d9}#root{position:relative;z-index:1;min-height:100vh}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:rgba(0,0,0,.05);border-radius:4px}::-webkit-scrollbar-thumb{background:rgba(0,0,0,.2);border-radius:4px}::-webkit-scrollbar-thumb:hover{background:rgba(0,0,0,.3)}.ant-layout{background:transparent!important}.ant-card{background:#ffffff!important;border:1px solid #f0f0f0}.stats-card{background:#ffffff;border-radius:8px;border:1px solid #f0f0f0;transition:all .3s cubic-bezier(.4,0,.2,1);overflow:hidden;position:relative}.stats-card:before{content:"";position:absolute;top:0;left:0;right:0;height:4px;background:linear-gradient(90deg,transparent,var(--primary-color),transparent);opacity:0;transition:opacity .3s}.stats-card:hover{transform:translateY(-4px);box-shadow:0 4px 12px #0000001a;border-color:#e8e8e8}.stats-card:hover:before{opacity:1}.chart-container{position:relative;height:100%;min-height:300px}.loading-spinner{display:inline-block;width:20px;height:20px;border:2px solid rgba(0,0,0,.1);border-top-color:var(--primary-color);border-radius:50%;animation:spin .8s linear infinite}@keyframes spin{to{transform:rotate(360deg)}}