jettask 0.2.15__py3-none-any.whl → 0.2.16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. jettask/__init__.py +14 -35
  2. jettask/{webui/__main__.py → __main__.py} +4 -4
  3. jettask/api/__init__.py +103 -0
  4. jettask/api/v1/__init__.py +29 -0
  5. jettask/api/v1/alerts.py +226 -0
  6. jettask/api/v1/analytics.py +323 -0
  7. jettask/api/v1/namespaces.py +134 -0
  8. jettask/api/v1/overview.py +136 -0
  9. jettask/api/v1/queues.py +530 -0
  10. jettask/api/v1/scheduled.py +420 -0
  11. jettask/api/v1/settings.py +44 -0
  12. jettask/{webui/api.py → api.py} +4 -46
  13. jettask/{webui/backend → backend}/main.py +21 -109
  14. jettask/{webui/backend → backend}/main_unified.py +1 -1
  15. jettask/{webui/backend → backend}/namespace_api_old.py +3 -30
  16. jettask/{webui/backend → backend}/namespace_data_access.py +2 -1
  17. jettask/{webui/backend → backend}/unified_api_router.py +14 -74
  18. jettask/{core/cli.py → cli.py} +106 -26
  19. jettask/config/nacos_config.py +386 -0
  20. jettask/core/app.py +8 -100
  21. jettask/core/db_manager.py +515 -0
  22. jettask/core/event_pool.py +5 -2
  23. jettask/core/unified_manager_base.py +47 -14
  24. jettask/{webui/db_init.py → db_init.py} +1 -1
  25. jettask/executors/asyncio.py +2 -2
  26. jettask/{webui/integrated_gradio_app.py → integrated_gradio_app.py} +1 -1
  27. jettask/{webui/multi_namespace_consumer.py → multi_namespace_consumer.py} +5 -2
  28. jettask/{webui/pg_consumer.py → pg_consumer.py} +137 -69
  29. jettask/{webui/run.py → run.py} +1 -1
  30. jettask/{webui/run_webui.py → run_webui.py} +4 -4
  31. jettask/scheduler/multi_namespace_scheduler.py +2 -2
  32. jettask/scheduler/unified_manager.py +5 -5
  33. jettask/scheduler/unified_scheduler_manager.py +1 -1
  34. jettask/schemas/__init__.py +166 -0
  35. jettask/schemas/alert.py +99 -0
  36. jettask/schemas/backlog.py +122 -0
  37. jettask/schemas/common.py +139 -0
  38. jettask/schemas/monitoring.py +181 -0
  39. jettask/schemas/namespace.py +168 -0
  40. jettask/schemas/queue.py +83 -0
  41. jettask/schemas/scheduled_task.py +128 -0
  42. jettask/schemas/task.py +70 -0
  43. jettask/services/__init__.py +24 -0
  44. jettask/services/alert_service.py +454 -0
  45. jettask/services/analytics_service.py +46 -0
  46. jettask/services/overview_service.py +978 -0
  47. jettask/services/queue_service.py +711 -0
  48. jettask/services/redis_monitor_service.py +151 -0
  49. jettask/services/scheduled_task_service.py +207 -0
  50. jettask/services/settings_service.py +758 -0
  51. jettask/services/task_service.py +157 -0
  52. jettask/{webui/task_center.py → task_center.py} +30 -8
  53. jettask/{webui/task_center_client.py → task_center_client.py} +1 -1
  54. jettask/{webui/config.py → webui_config.py} +6 -1
  55. jettask/webui_exceptions.py +67 -0
  56. jettask/webui_sql/verify_database.sql +72 -0
  57. {jettask-0.2.15.dist-info → jettask-0.2.16.dist-info}/METADATA +2 -1
  58. jettask-0.2.16.dist-info/RECORD +150 -0
  59. {jettask-0.2.15.dist-info → jettask-0.2.16.dist-info}/entry_points.txt +1 -1
  60. jettask/webui/backend/data_api.py +0 -3294
  61. jettask/webui/backend/namespace_api.py +0 -295
  62. jettask/webui/backend/queue_backlog_api.py +0 -727
  63. jettask/webui/backend/redis_monitor_api.py +0 -476
  64. jettask/webui/frontend/index.html +0 -13
  65. jettask/webui/frontend/package.json +0 -30
  66. jettask/webui/frontend/src/App.css +0 -109
  67. jettask/webui/frontend/src/App.jsx +0 -66
  68. jettask/webui/frontend/src/components/NamespaceSelector.jsx +0 -166
  69. jettask/webui/frontend/src/components/QueueBacklogChart.jsx +0 -298
  70. jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +0 -638
  71. jettask/webui/frontend/src/components/QueueDetailsTable.css +0 -65
  72. jettask/webui/frontend/src/components/QueueDetailsTable.jsx +0 -487
  73. jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +0 -465
  74. jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +0 -423
  75. jettask/webui/frontend/src/components/TaskFilter.jsx +0 -425
  76. jettask/webui/frontend/src/components/TimeRangeSelector.css +0 -21
  77. jettask/webui/frontend/src/components/TimeRangeSelector.jsx +0 -160
  78. jettask/webui/frontend/src/components/charts/QueueChart.jsx +0 -111
  79. jettask/webui/frontend/src/components/charts/QueueTrendChart.jsx +0 -115
  80. jettask/webui/frontend/src/components/charts/WorkerChart.jsx +0 -40
  81. jettask/webui/frontend/src/components/common/StatsCard.jsx +0 -18
  82. jettask/webui/frontend/src/components/layout/AppLayout.css +0 -95
  83. jettask/webui/frontend/src/components/layout/AppLayout.jsx +0 -49
  84. jettask/webui/frontend/src/components/layout/Header.css +0 -106
  85. jettask/webui/frontend/src/components/layout/Header.jsx +0 -106
  86. jettask/webui/frontend/src/components/layout/SideMenu.css +0 -137
  87. jettask/webui/frontend/src/components/layout/SideMenu.jsx +0 -209
  88. jettask/webui/frontend/src/components/layout/TabsNav.css +0 -244
  89. jettask/webui/frontend/src/components/layout/TabsNav.jsx +0 -206
  90. jettask/webui/frontend/src/components/layout/UserInfo.css +0 -197
  91. jettask/webui/frontend/src/components/layout/UserInfo.jsx +0 -197
  92. jettask/webui/frontend/src/contexts/LoadingContext.jsx +0 -27
  93. jettask/webui/frontend/src/contexts/NamespaceContext.jsx +0 -72
  94. jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +0 -245
  95. jettask/webui/frontend/src/index.css +0 -114
  96. jettask/webui/frontend/src/main.jsx +0 -22
  97. jettask/webui/frontend/src/pages/Alerts.jsx +0 -684
  98. jettask/webui/frontend/src/pages/Dashboard/index.css +0 -35
  99. jettask/webui/frontend/src/pages/Dashboard/index.jsx +0 -281
  100. jettask/webui/frontend/src/pages/Dashboard.jsx +0 -1330
  101. jettask/webui/frontend/src/pages/QueueDetail.jsx +0 -1117
  102. jettask/webui/frontend/src/pages/QueueMonitor.jsx +0 -527
  103. jettask/webui/frontend/src/pages/Queues.jsx +0 -12
  104. jettask/webui/frontend/src/pages/ScheduledTasks.jsx +0 -810
  105. jettask/webui/frontend/src/pages/Settings.jsx +0 -801
  106. jettask/webui/frontend/src/pages/Workers.jsx +0 -12
  107. jettask/webui/frontend/src/services/api.js +0 -159
  108. jettask/webui/frontend/src/services/queueTrend.js +0 -166
  109. jettask/webui/frontend/src/utils/suppressWarnings.js +0 -22
  110. jettask/webui/frontend/src/utils/userPreferences.js +0 -154
  111. jettask/webui/frontend/vite.config.js +0 -26
  112. jettask/webui/sql/init_database.sql +0 -640
  113. jettask-0.2.15.dist-info/RECORD +0 -172
  114. /jettask/{webui/backend → backend}/__init__.py +0 -0
  115. /jettask/{webui/backend → backend}/api/__init__.py +0 -0
  116. /jettask/{webui/backend → backend}/api/v1/__init__.py +0 -0
  117. /jettask/{webui/backend → backend}/api/v1/monitoring.py +0 -0
  118. /jettask/{webui/backend → backend}/api/v1/namespaces.py +0 -0
  119. /jettask/{webui/backend → backend}/api/v1/queues.py +0 -0
  120. /jettask/{webui/backend → backend}/api/v1/tasks.py +0 -0
  121. /jettask/{webui/backend → backend}/config.py +0 -0
  122. /jettask/{webui/backend → backend}/core/__init__.py +0 -0
  123. /jettask/{webui/backend → backend}/core/cache.py +0 -0
  124. /jettask/{webui/backend → backend}/core/database.py +0 -0
  125. /jettask/{webui/backend → backend}/core/exceptions.py +0 -0
  126. /jettask/{webui/backend → backend}/data_access.py +0 -0
  127. /jettask/{webui/backend → backend}/dependencies.py +0 -0
  128. /jettask/{webui/backend → backend}/init_meta_db.py +0 -0
  129. /jettask/{webui/backend → backend}/main_v2.py +0 -0
  130. /jettask/{webui/backend → backend}/models/__init__.py +0 -0
  131. /jettask/{webui/backend → backend}/models/requests.py +0 -0
  132. /jettask/{webui/backend → backend}/models/responses.py +0 -0
  133. /jettask/{webui/backend → backend}/queue_stats_v2.py +0 -0
  134. /jettask/{webui/backend → backend}/services/__init__.py +0 -0
  135. /jettask/{webui/backend → backend}/start.py +0 -0
  136. /jettask/{webui/cleanup_deprecated_tables.sql → cleanup_deprecated_tables.sql} +0 -0
  137. /jettask/{webui/gradio_app.py → gradio_app.py} +0 -0
  138. /jettask/{webui/__init__.py → main.py} +0 -0
  139. /jettask/{webui/models.py → models.py} +0 -0
  140. /jettask/{webui/run_monitor.py → run_monitor.py} +0 -0
  141. /jettask/{webui/schema.sql → schema.sql} +0 -0
  142. /jettask/{webui/unified_consumer_manager.py → unified_consumer_manager.py} +0 -0
  143. /jettask/{webui/models → webui_models}/__init__.py +0 -0
  144. /jettask/{webui/models → webui_models}/namespace.py +0 -0
  145. /jettask/{webui/sql → webui_sql}/batch_upsert_functions.sql +0 -0
  146. {jettask-0.2.15.dist-info → jettask-0.2.16.dist-info}/WHEEL +0 -0
  147. {jettask-0.2.15.dist-info → jettask-0.2.16.dist-info}/licenses/LICENSE +0 -0
  148. {jettask-0.2.15.dist-info → jettask-0.2.16.dist-info}/top_level.txt +0 -0
@@ -17,8 +17,8 @@ from datetime import datetime
17
17
  # 添加项目路径
18
18
  sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
19
19
 
20
- from jettask.webui.pg_consumer import PostgreSQLConsumer
21
- from jettask.webui.config import PostgreSQLConfig, RedisConfig
20
+ from jettask.pg_consumer import PostgreSQLConsumer
21
+ from jettask.webui_config import PostgreSQLConfig, RedisConfig
22
22
  from jettask.core.consumer_manager import ConsumerStrategy
23
23
  from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
24
24
  from sqlalchemy.orm import sessionmaker
@@ -268,6 +268,9 @@ class MultiNamespaceConsumerManager:
268
268
  if '/api/namespaces/' in base_url:
269
269
  # 从 http://localhost:8001/api/namespaces/default 提取 http://localhost:8001
270
270
  base_url = base_url.split('/api/namespaces/')[0]
271
+ elif '/api/v1/namespaces/' in base_url:
272
+ # 从 http://localhost:8001/api/v1/namespaces/default 提取 http://localhost:8001
273
+ base_url = base_url.split('/api/v1/namespaces/')[0]
271
274
 
272
275
  # 获取API服务的配置端点
273
276
  config_url = f"{base_url}/api/config"
@@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
17
17
  from sqlalchemy.orm import sessionmaker
18
18
  from sqlalchemy import text
19
19
 
20
- from jettask.webui.config import PostgreSQLConfig, RedisConfig
20
+ from jettask.webui_config import PostgreSQLConfig, RedisConfig
21
21
  from jettask.core.consumer_manager import ConsumerManager, ConsumerStrategy
22
22
  from jettask.core.offline_worker_recovery import OfflineWorkerRecovery
23
23
  from jettask.constants import is_internal_consumer, TASK_STATUS_PRIORITY
@@ -90,6 +90,10 @@ class PostgreSQLConsumer:
90
90
  self.backlog_monitor_lock_key = f"{prefix}:BACKLOG_MONITOR_LOCK" # 分布式锁键
91
91
  self.backlog_monitor_lock_ttl = backlog_monitor_interval * 2 # 锁的TTL(秒),设为采集间隔的2倍
92
92
 
93
+ # 队列注册表(替代scan命令)
94
+ self.queue_registry_key = f"{prefix}:QUEUE_REGISTRY" # 队列注册表的Redis key
95
+ self.stream_registry_key = f"{prefix}:STREAM_REGISTRY" # Stream注册表的Redis key(用于积压监控)
96
+
93
97
  async def start(self):
94
98
  """启动消费者"""
95
99
  logger.info(f"Starting PostgreSQL consumer (simplified) on node: {self.node_id}")
@@ -142,7 +146,7 @@ class PostgreSQLConsumer:
142
146
 
143
147
  # 创建SQLAlchemy异步引擎
144
148
  if self.pg_config.dsn.startswith('postgresql://'):
145
- dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+psycopg://', 1)
149
+ dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+asyncpg://', 1)
146
150
  else:
147
151
  dsn = self.pg_config.dsn
148
152
 
@@ -238,24 +242,39 @@ class PostgreSQLConsumer:
238
242
  logger.debug("PostgreSQL consumer stopped")
239
243
 
240
244
  async def _initial_queue_discovery(self):
241
- """初始队列发现,在启动时执行一次"""
245
+ """初始队列发现,在启动时执行一次 - 使用队列注册表替代scan"""
242
246
  try:
243
- pattern = f"{self.prefix}:QUEUE:*"
244
247
  new_queues = set()
245
- logger.info(f"Starting initial queue discovery with pattern: {pattern}")
248
+ logger.info(f"Starting initial queue discovery from queue registry: {self.queue_registry_key}")
246
249
 
247
- async for key in self.redis_client.scan_iter(match=pattern, count=100):
248
- # 从key中提取队列名,格式可能是:
249
- # - prefix:QUEUE:queue_name (普通队列)
250
- # - prefix:QUEUE:queue_name:priority (优先级队列)
251
- key_str = key.decode('utf-8')
252
- parts = key_str.split(":")
253
- if len(parts) >= 3:
254
- # 去掉前缀和QUEUE部分
255
- queue_parts = parts[2:] # 从第3部分开始是队列名
256
- queue_name = ":".join(queue_parts) # 重新组合,保留优先级部分
257
- new_queues.add(queue_name)
258
- logger.info(f"Found queue: {queue_name} from key: {key_str}")
250
+ # 从队列注册表获取所有队列
251
+ queue_members = await self.redis_client.smembers(self.queue_registry_key.encode())
252
+ for queue_name_bytes in queue_members:
253
+ queue_name = queue_name_bytes.decode('utf-8') if isinstance(queue_name_bytes, bytes) else str(queue_name_bytes)
254
+ new_queues.add(queue_name)
255
+ logger.info(f"Found registered queue: {queue_name}")
256
+
257
+ # 如果注册表为空,进行一次性的scan作为初始化(仅在首次运行时)
258
+ if not new_queues:
259
+ logger.warning(f"Queue registry is empty, performing one-time scan initialization...")
260
+ pattern = f"{self.prefix}:QUEUE:*"
261
+ async for key in self.redis_client.scan_iter(match=pattern, count=100):
262
+ key_str = key.decode('utf-8')
263
+ parts = key_str.split(":")
264
+ if len(parts) >= 3:
265
+ # 去掉前缀和QUEUE部分
266
+ queue_parts = parts[2:] # 从第3部分开始是队列名
267
+ queue_name = ":".join(queue_parts) # 重新组合,保留优先级部分
268
+ new_queues.add(queue_name)
269
+ logger.info(f"Found queue during scan: {queue_name} from key: {key_str}")
270
+
271
+ # 将发现的队列添加到注册表中
272
+ if new_queues:
273
+ pipeline = self.redis_client.pipeline()
274
+ for queue_name in new_queues:
275
+ pipeline.sadd(self.queue_registry_key.encode(), queue_name.encode())
276
+ await pipeline.execute()
277
+ logger.info(f"Registered {len(new_queues)} queues to registry during initialization")
259
278
 
260
279
  if new_queues:
261
280
  logger.info(f"Initial queue discovery found {len(new_queues)} queues: {new_queues}")
@@ -292,35 +311,34 @@ class PostgreSQLConsumer:
292
311
  logger.error(f"Error in initial queue discovery: {e}")
293
312
 
294
313
  async def _discover_queues(self):
295
- """定期发现新队列"""
314
+ """定期发现新队列 - 使用队列注册表替代scan"""
296
315
  while self._running:
297
316
  try:
298
- pattern = f"{self.prefix}:QUEUE:*"
299
317
  new_queues = set()
300
- # logger.info(f'{pattern=}')
301
- async for key in self.redis_client.scan_iter(match=pattern, count=100):
302
- # 从key中提取队列名,格式可能是:
303
- # - prefix:QUEUE:queue_name (普通队列)
304
- # - prefix:QUEUE:queue_name:priority (优先级队列)
305
- key_str = key.decode('utf-8')
306
- parts = key_str.split(":")
307
- if len(parts) >= 3:
308
- # 去掉前缀和QUEUE部分
309
- queue_parts = parts[2:] # 从第3部分开始是队列名
310
- queue_name = ":".join(queue_parts) # 重新组合,保留优先级部分
311
- new_queues.add(queue_name)
312
318
 
313
- # 为新发现的队列创建消费者组
314
- for queue in new_queues - self._known_queues:
315
- # 正确构建stream_key,保留优先级部分
316
- stream_key = f"{self.prefix}:QUEUE:{queue}"
317
- try:
318
- await self.redis_client.xgroup_create(
319
- stream_key, self.consumer_group, id='0', mkstream=True
320
- )
321
- logger.info(f"Created consumer group for new queue: {queue} with stream_key: {stream_key}")
322
- except redis.ResponseError:
323
- pass
319
+ # 从队列注册表获取所有队列
320
+ queue_members = await self.redis_client.smembers(self.queue_registry_key.encode())
321
+ for queue_name_bytes in queue_members:
322
+ queue_name = queue_name_bytes.decode('utf-8') if isinstance(queue_name_bytes, bytes) else str(queue_name_bytes)
323
+ new_queues.add(queue_name)
324
+
325
+ # 优化:添加日志,只在队列数量或内容发生变化时记录
326
+ if len(new_queues) != len(self._known_queues) or new_queues != self._known_queues:
327
+ logger.debug(f"Queue registry contains {len(new_queues)} queues: {sorted(new_queues)}")
328
+
329
+ # 为新发现的队列创建消费者组(注意:新队列应该通过生产者自动注册)
330
+ new_discovered = new_queues - self._known_queues
331
+ if new_discovered:
332
+ for queue in new_discovered:
333
+ # 正确构建stream_key,保留优先级部分
334
+ stream_key = f"{self.prefix}:QUEUE:{queue}"
335
+ try:
336
+ await self.redis_client.xgroup_create(
337
+ stream_key, self.consumer_group, id='0', mkstream=True
338
+ )
339
+ logger.info(f"Created consumer group for new queue: {queue} with stream_key: {stream_key}")
340
+ except redis.ResponseError:
341
+ pass
324
342
 
325
343
  # 更新ConsumerManager的队列列表(同步操作)
326
344
  if new_queues != self._known_queues:
@@ -353,7 +371,7 @@ class PostgreSQLConsumer:
353
371
  logger.error(f"Error updating worker queues: {e}")
354
372
 
355
373
  self._known_queues = new_queues
356
- await asyncio.sleep(30)
374
+ await asyncio.sleep(10) # 保持较短的检查间隔,确保新队列能及时发现
357
375
 
358
376
  except Exception as e:
359
377
  import traceback
@@ -1347,42 +1365,92 @@ class PostgreSQLConsumer:
1347
1365
  except Exception as e:
1348
1366
  logger.debug(f"No TASK_OFFSETS found for {task_offsets_key}: {e}")
1349
1367
 
1350
- # 使用SCAN命令扫描所有队列Stream(包括普通队列和优先级队列)
1368
+ # 使用Stream注册表替代SCAN命令获取队列信息
1351
1369
  stream_info_map = {} # {queue_name: [(stream_key, priority), ...]}
1352
- pattern = f"{self.prefix}:QUEUE:*".encode()
1353
- cursor = 0
1354
1370
 
1355
- # 使用SCAN命令,增大count参数以提高效率
1356
- while True:
1357
- cursor, keys = await self.redis_client.scan(cursor, match=pattern, count=10000)
1358
-
1359
- for key in keys:
1360
- key_str = key.decode()
1361
- # 移除前缀 "prefix:QUEUE:"
1362
- queue_part = key_str.replace(f"{self.prefix}:QUEUE:", "")
1363
-
1364
- # 检查是否是优先级队列(格式:queue_name:priority
1365
- parts = queue_part.split(':')
1371
+ # 从fredis中获取stream注册表(Hash结构)
1372
+ # 格式: {"queue_name:priority": "stream_key"}
1373
+ # 对于普通队列,priority为0
1374
+ stream_registry = await self.redis_client.hgetall(self.stream_registry_key.encode())
1375
+
1376
+ for queue_priority_bytes, stream_key_bytes in stream_registry.items():
1377
+ queue_priority_str = queue_priority_bytes.decode() if isinstance(queue_priority_bytes, bytes) else str(queue_priority_bytes)
1378
+ stream_key = stream_key_bytes.decode() if isinstance(stream_key_bytes, bytes) else str(stream_key_bytes)
1379
+
1380
+ # 解析queue_namepriority
1381
+ if ':' in queue_priority_str:
1382
+ parts = queue_priority_str.rsplit(':', 1)
1366
1383
  if len(parts) == 2 and parts[1].isdigit():
1367
- # 优先级队列
1368
1384
  queue_name = parts[0]
1369
1385
  priority = int(parts[1])
1386
+ else:
1387
+ # 如果最后一部分不是数字,说明是普通队列名包含冒号
1388
+ queue_name = queue_priority_str
1389
+ priority = 0
1390
+ else:
1391
+ # 普通队列
1392
+ queue_name = queue_priority_str
1393
+ priority = 0
1394
+
1395
+ if queue_name not in stream_info_map:
1396
+ stream_info_map[queue_name] = []
1397
+ stream_info_map[queue_name].append((stream_key, priority))
1398
+
1399
+ # 如果Stream注册表为空,进行一次性的scan作为初始化(仅在首次运行时)
1400
+ if not stream_info_map:
1401
+ logger.warning(f"Stream registry is empty, performing one-time scan initialization...")
1402
+ pattern = f"{self.prefix}:QUEUE:*".encode()
1403
+ cursor = 0
1404
+
1405
+ while True:
1406
+ cursor, keys = await self.redis_client.scan(cursor, match=pattern, count=10000)
1407
+
1408
+ for key in keys:
1409
+ key_str = key.decode()
1410
+ # 移除前缀 "prefix:QUEUE:"
1411
+ queue_part = key_str.replace(f"{self.prefix}:QUEUE:", "")
1412
+
1413
+ # 检查是否是优先级队列(格式:queue_name:priority)
1414
+ parts = queue_part.split(':')
1415
+ if len(parts) == 2 and parts[1].isdigit():
1416
+ # 优先级队列
1417
+ queue_name = parts[0]
1418
+ priority = int(parts[1])
1419
+ queue_priority_key = f"{queue_name}:{priority}"
1420
+ elif ':' not in queue_part:
1421
+ # 普通队列(不包含冒号)
1422
+ queue_name = queue_part
1423
+ priority = 0
1424
+ queue_priority_key = queue_name
1425
+ else:
1426
+ # 忽略其他格式的键(如消费组等)
1427
+ continue
1428
+
1370
1429
  if queue_name not in stream_info_map:
1371
1430
  stream_info_map[queue_name] = []
1372
1431
  stream_info_map[queue_name].append((key, priority))
1373
- elif ':' not in queue_part:
1374
- # 普通队列(不包含冒号)
1375
- queue_name = queue_part
1376
- if queue_name not in stream_info_map:
1377
- stream_info_map[queue_name] = []
1378
- stream_info_map[queue_name].append((key, 0)) # 普通队列优先级为0
1379
- # 忽略其他格式的键(如消费组等)
1380
-
1381
- if cursor == 0:
1382
- break
1432
+
1433
+ if cursor == 0:
1434
+ break
1435
+
1436
+ # 将发现的Stream信息添加到注册表中
1437
+ if stream_info_map:
1438
+ pipeline = self.redis_client.pipeline()
1439
+ for queue_name, stream_list in stream_info_map.items():
1440
+ for stream_key, priority in stream_list:
1441
+ if priority > 0:
1442
+ queue_priority_key = f"{queue_name}:{priority}"
1443
+ else:
1444
+ queue_priority_key = queue_name
1445
+ # stream_key已经是bytes类型(从scan_iter返回)
1446
+ if isinstance(stream_key, str):
1447
+ stream_key = stream_key.encode()
1448
+ pipeline.hset(self.stream_registry_key.encode(), queue_priority_key.encode(), stream_key)
1449
+ await pipeline.execute()
1450
+ logger.info(f"Registered {sum(len(stream_list) for stream_list in stream_info_map.values())} streams to registry during initialization")
1383
1451
 
1384
1452
  if not stream_info_map:
1385
- logger.debug("No streams found for backlog monitoring")
1453
+ logger.debug("No streams found in registry for backlog monitoring")
1386
1454
  return
1387
1455
 
1388
1456
  # 调试日志(使用debug级别避免刷屏)
@@ -11,7 +11,7 @@ import sys
11
11
  import os
12
12
  sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
13
13
 
14
- from jettask.webui.api import app, monitor
14
+ from jettask.api import app, monitor
15
15
  import uvicorn
16
16
 
17
17
  def main():
@@ -17,13 +17,13 @@ from pathlib import Path
17
17
  project_root = Path(__file__).parent.parent.parent
18
18
  sys.path.insert(0, str(project_root))
19
19
 
20
- from jettask.webui.config import PostgreSQLConfig
20
+ from jettask.webui_config import PostgreSQLConfig
21
21
 
22
22
 
23
23
  def run_fastapi_server(host: str = "0.0.0.0", port: int = 8000, with_consumer: bool = False):
24
24
  """运行FastAPI服务器"""
25
25
  import uvicorn
26
- from jettask.webui.api import app
26
+ from jettask.api import app
27
27
 
28
28
  # 如果需要启动消费者
29
29
  if with_consumer:
@@ -54,7 +54,7 @@ def run_gradio_interface(host: str = "0.0.0.0", port: int = 7860, share: bool =
54
54
  print("You can start it with: python -m jettask.webui.run_webui --mode fastapi")
55
55
  print()
56
56
 
57
- from jettask.webui.gradio_app import create_interface
57
+ from jettask.gradio_app import create_interface
58
58
 
59
59
  app = create_interface()
60
60
  print(f"Starting Gradio interface on http://{host}:{port}")
@@ -94,7 +94,7 @@ def run_combined_mode(host: str = "0.0.0.0", api_port: int = 8000, ui_port: int
94
94
 
95
95
  def run_integrated_gradio(host: str = "0.0.0.0", port: int = 7860, share: bool = False):
96
96
  """运行集成的Gradio界面(直接访问数据源)"""
97
- from jettask.webui.integrated_gradio_app import create_integrated_interface
97
+ from jettask.integrated_gradio_app import create_integrated_interface
98
98
 
99
99
  print("Starting Integrated Gradio interface (direct data access)...")
100
100
  print(f"Access the interface at http://{host}:{port}")
@@ -9,7 +9,7 @@ import traceback
9
9
  from typing import Dict, Set
10
10
 
11
11
  from jettask import Jettask
12
- from jettask.webui.task_center import TaskCenter
12
+ from jettask.task_center import TaskCenter
13
13
  from jettask.scheduler.scheduler import TaskScheduler
14
14
  from jettask.scheduler.manager import ScheduledTaskManager
15
15
 
@@ -219,7 +219,7 @@ def run_scheduler_for_namespace(namespace: str,
219
219
  scheduler_instance = None
220
220
  try:
221
221
  # 构建命名空间特定的URL
222
- task_center_url = f"{task_center_base_url}/api/namespaces/{namespace}"
222
+ task_center_url = f"{task_center_base_url}/api/v1/namespaces/{namespace}"
223
223
  logger.info(f"连接到任务中心: {task_center_url}")
224
224
 
225
225
  # 连接任务中心
@@ -11,7 +11,7 @@ import aiohttp
11
11
  import traceback
12
12
 
13
13
  from jettask import Jettask
14
- from jettask.webui.task_center import TaskCenter
14
+ from jettask.task_center import TaskCenter
15
15
  from .scheduler import TaskScheduler
16
16
  from .manager import ScheduledTaskManager
17
17
 
@@ -133,9 +133,9 @@ class UnifiedSchedulerManager:
133
133
  try:
134
134
  # 构建API URL
135
135
  if self.task_center_url.endswith('/api'):
136
- url = f"{self.task_center_url}/namespaces"
136
+ url = f"{self.task_center_url}/v1/namespaces"
137
137
  else:
138
- url = f"{self.task_center_url}/api/namespaces"
138
+ url = f"{self.task_center_url}/api/v1/namespaces"
139
139
 
140
140
  async with aiohttp.ClientSession() as session:
141
141
  async with session.get(url) as response:
@@ -223,9 +223,9 @@ class UnifiedSchedulerManager:
223
223
 
224
224
  # 构建命名空间URL
225
225
  if self.task_center_url.endswith('/api'):
226
- namespace_url = f"{self.task_center_url}/namespaces/{namespace}"
226
+ namespace_url = f"{self.task_center_url}/v1/namespaces/{namespace}"
227
227
  else:
228
- namespace_url = f"{self.task_center_url}/api/namespaces/{namespace}"
228
+ namespace_url = f"{self.task_center_url}/api/v1/namespaces/{namespace}"
229
229
 
230
230
  # 创建新进程
231
231
  process = multiprocessing.Process(
@@ -8,7 +8,7 @@ import multiprocessing
8
8
  from typing import Dict, Optional, Set
9
9
  from jettask.core.unified_manager_base import UnifiedManagerBase
10
10
  from jettask import Jettask
11
- from jettask.webui.task_center import TaskCenter
11
+ from jettask.task_center import TaskCenter
12
12
  from .scheduler import TaskScheduler
13
13
  from .manager import ScheduledTaskManager
14
14
 
@@ -0,0 +1,166 @@
1
+ """
2
+ 数据模型定义
3
+ 所有的Pydantic模型集中管理
4
+ """
5
+
6
+ # 任务相关模型
7
+ from .task import (
8
+ TasksRequest,
9
+ TaskDetailResponse,
10
+ TaskInfo,
11
+ TaskActionRequest,
12
+ TaskListResponse
13
+ )
14
+
15
+ # 队列相关模型
16
+ from .queue import (
17
+ TimeRangeQuery,
18
+ QueueStatsResponse,
19
+ QueueTimelineResponse,
20
+ TrimQueueRequest,
21
+ QueueInfo,
22
+ QueueStats,
23
+ QueueActionRequest
24
+ )
25
+
26
+ # 定时任务相关模型
27
+ from .scheduled_task import (
28
+ ScheduledTaskRequest,
29
+ ScheduledTaskResponse,
30
+ ScheduledTaskInfo,
31
+ ScheduleConfig,
32
+ ScheduledTaskCreate,
33
+ ScheduledTaskUpdate,
34
+ ScheduledTaskCreateRequest
35
+ )
36
+
37
+ # 告警相关模型
38
+ from .alert import (
39
+ AlertRuleRequest,
40
+ AlertRuleCreate,
41
+ AlertRuleUpdate,
42
+ AlertRule,
43
+ AlertInstance,
44
+ AlertSummary
45
+ )
46
+
47
+ # 命名空间相关模型
48
+ from .namespace import (
49
+ ConfigMode,
50
+ NamespaceCreate,
51
+ NamespaceUpdate,
52
+ NamespaceResponse,
53
+ NamespaceInfo,
54
+ NamespaceCreateRequest,
55
+ NamespaceUpdateRequest
56
+ )
57
+
58
+ # 积压监控相关模型
59
+ from .backlog import (
60
+ BacklogTrendRequest,
61
+ BacklogSnapshot,
62
+ BacklogStatistics,
63
+ BacklogTrendResponse,
64
+ BacklogAlert
65
+ )
66
+
67
+ # 通用模型
68
+ from .common import (
69
+ ErrorResponse,
70
+ FilterCondition,
71
+ BaseListRequest,
72
+ TimeRangeRequest,
73
+ BatchOperationRequest,
74
+ BatchOperationResponse,
75
+ PaginationResponse,
76
+ ListResponse,
77
+ HealthCheck,
78
+ SystemConfigUpdateRequest,
79
+ ApiResponse
80
+ )
81
+
82
+ # 监控和分析模型
83
+ from .monitoring import (
84
+ MetricPoint,
85
+ TimeSeries,
86
+ MonitoringMetrics,
87
+ AnalyticsData,
88
+ SystemHealth,
89
+ DashboardOverviewRequest,
90
+ DashboardOverview,
91
+ WorkerMetrics
92
+ )
93
+
94
+ __all__ = [
95
+ # Task
96
+ 'TasksRequest',
97
+ 'TaskDetailResponse',
98
+ 'TaskInfo',
99
+ 'TaskActionRequest',
100
+ 'TaskListResponse',
101
+
102
+ # Queue
103
+ 'TimeRangeQuery',
104
+ 'QueueStatsResponse',
105
+ 'QueueTimelineResponse',
106
+ 'TrimQueueRequest',
107
+ 'QueueInfo',
108
+ 'QueueStats',
109
+ 'QueueActionRequest',
110
+
111
+ # Scheduled Task
112
+ 'ScheduledTaskRequest',
113
+ 'ScheduledTaskResponse',
114
+ 'ScheduledTaskInfo',
115
+ 'ScheduleConfig',
116
+ 'ScheduledTaskCreate',
117
+ 'ScheduledTaskUpdate',
118
+ 'ScheduledTaskCreateRequest',
119
+
120
+ # Alert
121
+ 'AlertRuleRequest',
122
+ 'AlertRuleCreate',
123
+ 'AlertRuleUpdate',
124
+ 'AlertRule',
125
+ 'AlertInstance',
126
+ 'AlertSummary',
127
+
128
+ # Namespace
129
+ 'ConfigMode',
130
+ 'NamespaceCreate',
131
+ 'NamespaceUpdate',
132
+ 'NamespaceResponse',
133
+ 'NamespaceInfo',
134
+ 'NamespaceCreateRequest',
135
+ 'NamespaceUpdateRequest',
136
+
137
+ # Backlog
138
+ 'BacklogTrendRequest',
139
+ 'BacklogSnapshot',
140
+ 'BacklogStatistics',
141
+ 'BacklogTrendResponse',
142
+ 'BacklogAlert',
143
+
144
+ # Common
145
+ 'ErrorResponse',
146
+ 'FilterCondition',
147
+ 'BaseListRequest',
148
+ 'TimeRangeRequest',
149
+ 'BatchOperationRequest',
150
+ 'BatchOperationResponse',
151
+ 'PaginationResponse',
152
+ 'ListResponse',
153
+ 'HealthCheck',
154
+ 'SystemConfigUpdateRequest',
155
+ 'ApiResponse',
156
+
157
+ # Monitoring
158
+ 'MetricPoint',
159
+ 'TimeSeries',
160
+ 'MonitoringMetrics',
161
+ 'AnalyticsData',
162
+ 'SystemHealth',
163
+ 'DashboardOverviewRequest',
164
+ 'DashboardOverview',
165
+ 'WorkerMetrics'
166
+ ]
@@ -0,0 +1,99 @@
1
+ """
2
+ 告警相关的数据模型
3
+ """
4
+ from typing import Optional, List, Dict, Any
5
+ from datetime import datetime
6
+ from pydantic import BaseModel, Field
7
+
8
+
9
+ class AlertRuleRequest(BaseModel):
10
+ """告警规则请求模型"""
11
+ rule_name: str = Field(..., description="规则名称")
12
+ metric_type: str = Field(..., description="指标类型")
13
+ threshold: float = Field(..., description="阈值")
14
+ operator: str = Field(..., description="比较操作符", pattern="^(gt|gte|lt|lte|eq|ne)$")
15
+ duration: int = Field(..., ge=1, description="持续时间(秒)")
16
+ severity: str = Field(..., description="严重级别", pattern="^(low|medium|high|critical)$")
17
+ enabled: bool = Field(default=True, description="是否启用")
18
+ description: Optional[str] = Field(None, description="规则描述")
19
+
20
+
21
+ class AlertRuleCreate(BaseModel):
22
+ """创建告警规则请求模型"""
23
+ name: str = Field(..., description="规则名称")
24
+ description: Optional[str] = Field(None, description="规则描述")
25
+ metric_type: str = Field(..., description="监控指标类型")
26
+ threshold: float = Field(..., description="告警阈值")
27
+ comparison_operator: str = Field(..., description="比较操作符")
28
+ evaluation_window: int = Field(..., description="评估窗口(秒)")
29
+ severity: str = Field(..., description="告警级别")
30
+ notification_channels: List[str] = Field(default=[], description="通知渠道")
31
+ enabled: bool = Field(default=True, description="是否启用")
32
+
33
+
34
+ class AlertRuleUpdate(BaseModel):
35
+ """更新告警规则请求模型"""
36
+ name: Optional[str] = Field(None, description="规则名称")
37
+ description: Optional[str] = Field(None, description="规则描述")
38
+ metric_type: Optional[str] = Field(None, description="监控指标类型")
39
+ threshold: Optional[float] = Field(None, description="告警阈值")
40
+ comparison_operator: Optional[str] = Field(None, description="比较操作符")
41
+ evaluation_window: Optional[int] = Field(None, description="评估窗口(秒)")
42
+ severity: Optional[str] = Field(None, description="告警级别")
43
+ notification_channels: Optional[List[str]] = Field(None, description="通知渠道")
44
+ enabled: Optional[bool] = Field(None, description="是否启用")
45
+
46
+
47
+ class AlertRule(BaseModel):
48
+ """告警规则模型"""
49
+ id: str = Field(..., description="规则ID")
50
+ name: str = Field(..., description="规则名称")
51
+ description: Optional[str] = Field(None, description="规则描述")
52
+ metric_type: str = Field(..., description="监控指标类型")
53
+ threshold: float = Field(..., description="告警阈值")
54
+ comparison_operator: str = Field(..., description="比较操作符")
55
+ evaluation_window: int = Field(..., description="评估窗口(秒)")
56
+ severity: str = Field(..., description="告警级别")
57
+ notification_channels: List[str] = Field(default=[], description="通知渠道")
58
+ enabled: bool = Field(default=True, description="是否启用")
59
+ created_at: datetime = Field(..., description="创建时间")
60
+ updated_at: Optional[datetime] = Field(None, description="更新时间")
61
+
62
+ class Config:
63
+ json_encoders = {
64
+ datetime: lambda v: v.isoformat() if v else None
65
+ }
66
+
67
+
68
+ class AlertInstance(BaseModel):
69
+ """告警实例模型"""
70
+ id: str = Field(..., description="告警实例ID")
71
+ rule_id: str = Field(..., description="规则ID")
72
+ rule_name: str = Field(..., description="规则名称")
73
+ status: str = Field(..., description="告警状态", pattern="^(active|resolved|suppressed)$")
74
+ severity: str = Field(..., description="告警级别")
75
+ message: str = Field(..., description="告警消息")
76
+ metric_value: float = Field(..., description="当前指标值")
77
+ threshold: float = Field(..., description="阈值")
78
+ started_at: datetime = Field(..., description="开始时间")
79
+ resolved_at: Optional[datetime] = Field(None, description="解决时间")
80
+ acknowledged_at: Optional[datetime] = Field(None, description="确认时间")
81
+ acknowledged_by: Optional[str] = Field(None, description="确认人")
82
+ namespace: str = Field(..., description="命名空间")
83
+ labels: Dict[str, str] = Field(default={}, description="标签")
84
+
85
+ class Config:
86
+ json_encoders = {
87
+ datetime: lambda v: v.isoformat() if v else None
88
+ }
89
+
90
+
91
+ class AlertSummary(BaseModel):
92
+ """告警摘要模型"""
93
+ total_alerts: int = Field(default=0, description="总告警数")
94
+ active_alerts: int = Field(default=0, description="活跃告警数")
95
+ resolved_alerts: int = Field(default=0, description="已解决告警数")
96
+ critical_alerts: int = Field(default=0, description="严重告警数")
97
+ high_alerts: int = Field(default=0, description="高级告警数")
98
+ medium_alerts: int = Field(default=0, description="中级告警数")
99
+ low_alerts: int = Field(default=0, description="低级告警数")