jettask 0.2.15__py3-none-any.whl → 0.2.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/__init__.py +14 -35
- jettask/{webui/__main__.py → __main__.py} +4 -4
- jettask/api/__init__.py +103 -0
- jettask/api/v1/__init__.py +29 -0
- jettask/api/v1/alerts.py +226 -0
- jettask/api/v1/analytics.py +323 -0
- jettask/api/v1/namespaces.py +134 -0
- jettask/api/v1/overview.py +136 -0
- jettask/api/v1/queues.py +530 -0
- jettask/api/v1/scheduled.py +420 -0
- jettask/api/v1/settings.py +44 -0
- jettask/{webui/api.py → api.py} +4 -46
- jettask/{webui/backend → backend}/main.py +21 -109
- jettask/{webui/backend → backend}/main_unified.py +1 -1
- jettask/{webui/backend → backend}/namespace_api_old.py +3 -30
- jettask/{webui/backend → backend}/namespace_data_access.py +2 -1
- jettask/{webui/backend → backend}/unified_api_router.py +14 -74
- jettask/{core/cli.py → cli.py} +106 -26
- jettask/config/nacos_config.py +386 -0
- jettask/core/app.py +8 -100
- jettask/core/db_manager.py +515 -0
- jettask/core/event_pool.py +5 -2
- jettask/core/unified_manager_base.py +59 -14
- jettask/{webui/db_init.py → db_init.py} +1 -1
- jettask/executors/asyncio.py +2 -2
- jettask/{webui/integrated_gradio_app.py → integrated_gradio_app.py} +1 -1
- jettask/{webui/multi_namespace_consumer.py → multi_namespace_consumer.py} +5 -2
- jettask/{webui/pg_consumer.py → pg_consumer.py} +137 -69
- jettask/{webui/run.py → run.py} +1 -1
- jettask/{webui/run_webui.py → run_webui.py} +4 -4
- jettask/scheduler/manager.py +6 -0
- jettask/scheduler/multi_namespace_scheduler.py +2 -2
- jettask/scheduler/unified_manager.py +5 -5
- jettask/scheduler/unified_scheduler_manager.py +20 -12
- jettask/schemas/__init__.py +166 -0
- jettask/schemas/alert.py +99 -0
- jettask/schemas/backlog.py +122 -0
- jettask/schemas/common.py +139 -0
- jettask/schemas/monitoring.py +181 -0
- jettask/schemas/namespace.py +168 -0
- jettask/schemas/queue.py +83 -0
- jettask/schemas/scheduled_task.py +128 -0
- jettask/schemas/task.py +70 -0
- jettask/services/__init__.py +24 -0
- jettask/services/alert_service.py +454 -0
- jettask/services/analytics_service.py +46 -0
- jettask/services/overview_service.py +978 -0
- jettask/services/queue_service.py +711 -0
- jettask/services/redis_monitor_service.py +151 -0
- jettask/services/scheduled_task_service.py +207 -0
- jettask/services/settings_service.py +758 -0
- jettask/services/task_service.py +157 -0
- jettask/{webui/task_center.py → task_center.py} +30 -8
- jettask/{webui/task_center_client.py → task_center_client.py} +1 -1
- jettask/{webui/config.py → webui_config.py} +6 -1
- jettask/webui_exceptions.py +67 -0
- jettask/webui_sql/verify_database.sql +72 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/METADATA +2 -1
- jettask-0.2.17.dist-info/RECORD +150 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/entry_points.txt +1 -1
- jettask/webui/backend/data_api.py +0 -3294
- jettask/webui/backend/namespace_api.py +0 -295
- jettask/webui/backend/queue_backlog_api.py +0 -727
- jettask/webui/backend/redis_monitor_api.py +0 -476
- jettask/webui/frontend/index.html +0 -13
- jettask/webui/frontend/package.json +0 -30
- jettask/webui/frontend/src/App.css +0 -109
- jettask/webui/frontend/src/App.jsx +0 -66
- jettask/webui/frontend/src/components/NamespaceSelector.jsx +0 -166
- jettask/webui/frontend/src/components/QueueBacklogChart.jsx +0 -298
- jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +0 -638
- jettask/webui/frontend/src/components/QueueDetailsTable.css +0 -65
- jettask/webui/frontend/src/components/QueueDetailsTable.jsx +0 -487
- jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +0 -465
- jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +0 -423
- jettask/webui/frontend/src/components/TaskFilter.jsx +0 -425
- jettask/webui/frontend/src/components/TimeRangeSelector.css +0 -21
- jettask/webui/frontend/src/components/TimeRangeSelector.jsx +0 -160
- jettask/webui/frontend/src/components/charts/QueueChart.jsx +0 -111
- jettask/webui/frontend/src/components/charts/QueueTrendChart.jsx +0 -115
- jettask/webui/frontend/src/components/charts/WorkerChart.jsx +0 -40
- jettask/webui/frontend/src/components/common/StatsCard.jsx +0 -18
- jettask/webui/frontend/src/components/layout/AppLayout.css +0 -95
- jettask/webui/frontend/src/components/layout/AppLayout.jsx +0 -49
- jettask/webui/frontend/src/components/layout/Header.css +0 -106
- jettask/webui/frontend/src/components/layout/Header.jsx +0 -106
- jettask/webui/frontend/src/components/layout/SideMenu.css +0 -137
- jettask/webui/frontend/src/components/layout/SideMenu.jsx +0 -209
- jettask/webui/frontend/src/components/layout/TabsNav.css +0 -244
- jettask/webui/frontend/src/components/layout/TabsNav.jsx +0 -206
- jettask/webui/frontend/src/components/layout/UserInfo.css +0 -197
- jettask/webui/frontend/src/components/layout/UserInfo.jsx +0 -197
- jettask/webui/frontend/src/contexts/LoadingContext.jsx +0 -27
- jettask/webui/frontend/src/contexts/NamespaceContext.jsx +0 -72
- jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +0 -245
- jettask/webui/frontend/src/index.css +0 -114
- jettask/webui/frontend/src/main.jsx +0 -22
- jettask/webui/frontend/src/pages/Alerts.jsx +0 -684
- jettask/webui/frontend/src/pages/Dashboard/index.css +0 -35
- jettask/webui/frontend/src/pages/Dashboard/index.jsx +0 -281
- jettask/webui/frontend/src/pages/Dashboard.jsx +0 -1330
- jettask/webui/frontend/src/pages/QueueDetail.jsx +0 -1117
- jettask/webui/frontend/src/pages/QueueMonitor.jsx +0 -527
- jettask/webui/frontend/src/pages/Queues.jsx +0 -12
- jettask/webui/frontend/src/pages/ScheduledTasks.jsx +0 -810
- jettask/webui/frontend/src/pages/Settings.jsx +0 -801
- jettask/webui/frontend/src/pages/Workers.jsx +0 -12
- jettask/webui/frontend/src/services/api.js +0 -159
- jettask/webui/frontend/src/services/queueTrend.js +0 -166
- jettask/webui/frontend/src/utils/suppressWarnings.js +0 -22
- jettask/webui/frontend/src/utils/userPreferences.js +0 -154
- jettask/webui/frontend/vite.config.js +0 -26
- jettask/webui/sql/init_database.sql +0 -640
- jettask-0.2.15.dist-info/RECORD +0 -172
- /jettask/{webui/backend → backend}/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/monitoring.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/namespaces.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/queues.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/tasks.py +0 -0
- /jettask/{webui/backend → backend}/config.py +0 -0
- /jettask/{webui/backend → backend}/core/__init__.py +0 -0
- /jettask/{webui/backend → backend}/core/cache.py +0 -0
- /jettask/{webui/backend → backend}/core/database.py +0 -0
- /jettask/{webui/backend → backend}/core/exceptions.py +0 -0
- /jettask/{webui/backend → backend}/data_access.py +0 -0
- /jettask/{webui/backend → backend}/dependencies.py +0 -0
- /jettask/{webui/backend → backend}/init_meta_db.py +0 -0
- /jettask/{webui/backend → backend}/main_v2.py +0 -0
- /jettask/{webui/backend → backend}/models/__init__.py +0 -0
- /jettask/{webui/backend → backend}/models/requests.py +0 -0
- /jettask/{webui/backend → backend}/models/responses.py +0 -0
- /jettask/{webui/backend → backend}/queue_stats_v2.py +0 -0
- /jettask/{webui/backend → backend}/services/__init__.py +0 -0
- /jettask/{webui/backend → backend}/start.py +0 -0
- /jettask/{webui/cleanup_deprecated_tables.sql → cleanup_deprecated_tables.sql} +0 -0
- /jettask/{webui/gradio_app.py → gradio_app.py} +0 -0
- /jettask/{webui/__init__.py → main.py} +0 -0
- /jettask/{webui/models.py → models.py} +0 -0
- /jettask/{webui/run_monitor.py → run_monitor.py} +0 -0
- /jettask/{webui/schema.sql → schema.sql} +0 -0
- /jettask/{webui/unified_consumer_manager.py → unified_consumer_manager.py} +0 -0
- /jettask/{webui/models → webui_models}/__init__.py +0 -0
- /jettask/{webui/models → webui_models}/namespace.py +0 -0
- /jettask/{webui/sql → webui_sql}/batch_upsert_functions.sql +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/WHEEL +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/top_level.txt +0 -0
|
@@ -14,7 +14,7 @@ from sqlalchemy import text
|
|
|
14
14
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
15
15
|
from sqlalchemy.orm import sessionmaker
|
|
16
16
|
|
|
17
|
-
from jettask.
|
|
17
|
+
from jettask.webui_config import RedisConfig, PostgreSQLConfig
|
|
18
18
|
|
|
19
19
|
# 设置日志
|
|
20
20
|
logging.basicConfig(level=logging.INFO)
|
|
@@ -17,8 +17,8 @@ from datetime import datetime
|
|
|
17
17
|
# 添加项目路径
|
|
18
18
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
19
19
|
|
|
20
|
-
from jettask.
|
|
21
|
-
from jettask.
|
|
20
|
+
from jettask.pg_consumer import PostgreSQLConsumer
|
|
21
|
+
from jettask.webui_config import PostgreSQLConfig, RedisConfig
|
|
22
22
|
from jettask.core.consumer_manager import ConsumerStrategy
|
|
23
23
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
24
24
|
from sqlalchemy.orm import sessionmaker
|
|
@@ -268,6 +268,9 @@ class MultiNamespaceConsumerManager:
|
|
|
268
268
|
if '/api/namespaces/' in base_url:
|
|
269
269
|
# 从 http://localhost:8001/api/namespaces/default 提取 http://localhost:8001
|
|
270
270
|
base_url = base_url.split('/api/namespaces/')[0]
|
|
271
|
+
elif '/api/v1/namespaces/' in base_url:
|
|
272
|
+
# 从 http://localhost:8001/api/v1/namespaces/default 提取 http://localhost:8001
|
|
273
|
+
base_url = base_url.split('/api/v1/namespaces/')[0]
|
|
271
274
|
|
|
272
275
|
# 获取API服务的配置端点
|
|
273
276
|
config_url = f"{base_url}/api/config"
|
|
@@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
|
17
17
|
from sqlalchemy.orm import sessionmaker
|
|
18
18
|
from sqlalchemy import text
|
|
19
19
|
|
|
20
|
-
from jettask.
|
|
20
|
+
from jettask.webui_config import PostgreSQLConfig, RedisConfig
|
|
21
21
|
from jettask.core.consumer_manager import ConsumerManager, ConsumerStrategy
|
|
22
22
|
from jettask.core.offline_worker_recovery import OfflineWorkerRecovery
|
|
23
23
|
from jettask.constants import is_internal_consumer, TASK_STATUS_PRIORITY
|
|
@@ -90,6 +90,10 @@ class PostgreSQLConsumer:
|
|
|
90
90
|
self.backlog_monitor_lock_key = f"{prefix}:BACKLOG_MONITOR_LOCK" # 分布式锁键
|
|
91
91
|
self.backlog_monitor_lock_ttl = backlog_monitor_interval * 2 # 锁的TTL(秒),设为采集间隔的2倍
|
|
92
92
|
|
|
93
|
+
# 队列注册表(替代scan命令)
|
|
94
|
+
self.queue_registry_key = f"{prefix}:QUEUE_REGISTRY" # 队列注册表的Redis key
|
|
95
|
+
self.stream_registry_key = f"{prefix}:STREAM_REGISTRY" # Stream注册表的Redis key(用于积压监控)
|
|
96
|
+
|
|
93
97
|
async def start(self):
|
|
94
98
|
"""启动消费者"""
|
|
95
99
|
logger.info(f"Starting PostgreSQL consumer (simplified) on node: {self.node_id}")
|
|
@@ -142,7 +146,7 @@ class PostgreSQLConsumer:
|
|
|
142
146
|
|
|
143
147
|
# 创建SQLAlchemy异步引擎
|
|
144
148
|
if self.pg_config.dsn.startswith('postgresql://'):
|
|
145
|
-
dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+
|
|
149
|
+
dsn = self.pg_config.dsn.replace('postgresql://', 'postgresql+asyncpg://', 1)
|
|
146
150
|
else:
|
|
147
151
|
dsn = self.pg_config.dsn
|
|
148
152
|
|
|
@@ -238,24 +242,39 @@ class PostgreSQLConsumer:
|
|
|
238
242
|
logger.debug("PostgreSQL consumer stopped")
|
|
239
243
|
|
|
240
244
|
async def _initial_queue_discovery(self):
|
|
241
|
-
"""初始队列发现,在启动时执行一次"""
|
|
245
|
+
"""初始队列发现,在启动时执行一次 - 使用队列注册表替代scan"""
|
|
242
246
|
try:
|
|
243
|
-
pattern = f"{self.prefix}:QUEUE:*"
|
|
244
247
|
new_queues = set()
|
|
245
|
-
logger.info(f"Starting initial queue discovery
|
|
248
|
+
logger.info(f"Starting initial queue discovery from queue registry: {self.queue_registry_key}")
|
|
246
249
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
250
|
+
# 从队列注册表获取所有队列
|
|
251
|
+
queue_members = await self.redis_client.smembers(self.queue_registry_key.encode())
|
|
252
|
+
for queue_name_bytes in queue_members:
|
|
253
|
+
queue_name = queue_name_bytes.decode('utf-8') if isinstance(queue_name_bytes, bytes) else str(queue_name_bytes)
|
|
254
|
+
new_queues.add(queue_name)
|
|
255
|
+
logger.info(f"Found registered queue: {queue_name}")
|
|
256
|
+
|
|
257
|
+
# 如果注册表为空,进行一次性的scan作为初始化(仅在首次运行时)
|
|
258
|
+
if not new_queues:
|
|
259
|
+
logger.warning(f"Queue registry is empty, performing one-time scan initialization...")
|
|
260
|
+
pattern = f"{self.prefix}:QUEUE:*"
|
|
261
|
+
async for key in self.redis_client.scan_iter(match=pattern, count=100):
|
|
262
|
+
key_str = key.decode('utf-8')
|
|
263
|
+
parts = key_str.split(":")
|
|
264
|
+
if len(parts) >= 3:
|
|
265
|
+
# 去掉前缀和QUEUE部分
|
|
266
|
+
queue_parts = parts[2:] # 从第3部分开始是队列名
|
|
267
|
+
queue_name = ":".join(queue_parts) # 重新组合,保留优先级部分
|
|
268
|
+
new_queues.add(queue_name)
|
|
269
|
+
logger.info(f"Found queue during scan: {queue_name} from key: {key_str}")
|
|
270
|
+
|
|
271
|
+
# 将发现的队列添加到注册表中
|
|
272
|
+
if new_queues:
|
|
273
|
+
pipeline = self.redis_client.pipeline()
|
|
274
|
+
for queue_name in new_queues:
|
|
275
|
+
pipeline.sadd(self.queue_registry_key.encode(), queue_name.encode())
|
|
276
|
+
await pipeline.execute()
|
|
277
|
+
logger.info(f"Registered {len(new_queues)} queues to registry during initialization")
|
|
259
278
|
|
|
260
279
|
if new_queues:
|
|
261
280
|
logger.info(f"Initial queue discovery found {len(new_queues)} queues: {new_queues}")
|
|
@@ -292,35 +311,34 @@ class PostgreSQLConsumer:
|
|
|
292
311
|
logger.error(f"Error in initial queue discovery: {e}")
|
|
293
312
|
|
|
294
313
|
async def _discover_queues(self):
|
|
295
|
-
"""定期发现新队列"""
|
|
314
|
+
"""定期发现新队列 - 使用队列注册表替代scan"""
|
|
296
315
|
while self._running:
|
|
297
316
|
try:
|
|
298
|
-
pattern = f"{self.prefix}:QUEUE:*"
|
|
299
317
|
new_queues = set()
|
|
300
|
-
# logger.info(f'{pattern=}')
|
|
301
|
-
async for key in self.redis_client.scan_iter(match=pattern, count=100):
|
|
302
|
-
# 从key中提取队列名,格式可能是:
|
|
303
|
-
# - prefix:QUEUE:queue_name (普通队列)
|
|
304
|
-
# - prefix:QUEUE:queue_name:priority (优先级队列)
|
|
305
|
-
key_str = key.decode('utf-8')
|
|
306
|
-
parts = key_str.split(":")
|
|
307
|
-
if len(parts) >= 3:
|
|
308
|
-
# 去掉前缀和QUEUE部分
|
|
309
|
-
queue_parts = parts[2:] # 从第3部分开始是队列名
|
|
310
|
-
queue_name = ":".join(queue_parts) # 重新组合,保留优先级部分
|
|
311
|
-
new_queues.add(queue_name)
|
|
312
318
|
|
|
313
|
-
#
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
319
|
+
# 从队列注册表获取所有队列
|
|
320
|
+
queue_members = await self.redis_client.smembers(self.queue_registry_key.encode())
|
|
321
|
+
for queue_name_bytes in queue_members:
|
|
322
|
+
queue_name = queue_name_bytes.decode('utf-8') if isinstance(queue_name_bytes, bytes) else str(queue_name_bytes)
|
|
323
|
+
new_queues.add(queue_name)
|
|
324
|
+
|
|
325
|
+
# 优化:添加日志,只在队列数量或内容发生变化时记录
|
|
326
|
+
if len(new_queues) != len(self._known_queues) or new_queues != self._known_queues:
|
|
327
|
+
logger.debug(f"Queue registry contains {len(new_queues)} queues: {sorted(new_queues)}")
|
|
328
|
+
|
|
329
|
+
# 为新发现的队列创建消费者组(注意:新队列应该通过生产者自动注册)
|
|
330
|
+
new_discovered = new_queues - self._known_queues
|
|
331
|
+
if new_discovered:
|
|
332
|
+
for queue in new_discovered:
|
|
333
|
+
# 正确构建stream_key,保留优先级部分
|
|
334
|
+
stream_key = f"{self.prefix}:QUEUE:{queue}"
|
|
335
|
+
try:
|
|
336
|
+
await self.redis_client.xgroup_create(
|
|
337
|
+
stream_key, self.consumer_group, id='0', mkstream=True
|
|
338
|
+
)
|
|
339
|
+
logger.info(f"Created consumer group for new queue: {queue} with stream_key: {stream_key}")
|
|
340
|
+
except redis.ResponseError:
|
|
341
|
+
pass
|
|
324
342
|
|
|
325
343
|
# 更新ConsumerManager的队列列表(同步操作)
|
|
326
344
|
if new_queues != self._known_queues:
|
|
@@ -353,7 +371,7 @@ class PostgreSQLConsumer:
|
|
|
353
371
|
logger.error(f"Error updating worker queues: {e}")
|
|
354
372
|
|
|
355
373
|
self._known_queues = new_queues
|
|
356
|
-
await asyncio.sleep(
|
|
374
|
+
await asyncio.sleep(10) # 保持较短的检查间隔,确保新队列能及时发现
|
|
357
375
|
|
|
358
376
|
except Exception as e:
|
|
359
377
|
import traceback
|
|
@@ -1347,42 +1365,92 @@ class PostgreSQLConsumer:
|
|
|
1347
1365
|
except Exception as e:
|
|
1348
1366
|
logger.debug(f"No TASK_OFFSETS found for {task_offsets_key}: {e}")
|
|
1349
1367
|
|
|
1350
|
-
# 使用SCAN
|
|
1368
|
+
# 使用Stream注册表替代SCAN命令获取队列信息
|
|
1351
1369
|
stream_info_map = {} # {queue_name: [(stream_key, priority), ...]}
|
|
1352
|
-
pattern = f"{self.prefix}:QUEUE:*".encode()
|
|
1353
|
-
cursor = 0
|
|
1354
1370
|
|
|
1355
|
-
#
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1371
|
+
# 从fredis中获取stream注册表(Hash结构)
|
|
1372
|
+
# 格式: {"queue_name:priority": "stream_key"}
|
|
1373
|
+
# 对于普通队列,priority为0
|
|
1374
|
+
stream_registry = await self.redis_client.hgetall(self.stream_registry_key.encode())
|
|
1375
|
+
|
|
1376
|
+
for queue_priority_bytes, stream_key_bytes in stream_registry.items():
|
|
1377
|
+
queue_priority_str = queue_priority_bytes.decode() if isinstance(queue_priority_bytes, bytes) else str(queue_priority_bytes)
|
|
1378
|
+
stream_key = stream_key_bytes.decode() if isinstance(stream_key_bytes, bytes) else str(stream_key_bytes)
|
|
1379
|
+
|
|
1380
|
+
# 解析queue_name和priority
|
|
1381
|
+
if ':' in queue_priority_str:
|
|
1382
|
+
parts = queue_priority_str.rsplit(':', 1)
|
|
1366
1383
|
if len(parts) == 2 and parts[1].isdigit():
|
|
1367
|
-
# 优先级队列
|
|
1368
1384
|
queue_name = parts[0]
|
|
1369
1385
|
priority = int(parts[1])
|
|
1386
|
+
else:
|
|
1387
|
+
# 如果最后一部分不是数字,说明是普通队列名包含冒号
|
|
1388
|
+
queue_name = queue_priority_str
|
|
1389
|
+
priority = 0
|
|
1390
|
+
else:
|
|
1391
|
+
# 普通队列
|
|
1392
|
+
queue_name = queue_priority_str
|
|
1393
|
+
priority = 0
|
|
1394
|
+
|
|
1395
|
+
if queue_name not in stream_info_map:
|
|
1396
|
+
stream_info_map[queue_name] = []
|
|
1397
|
+
stream_info_map[queue_name].append((stream_key, priority))
|
|
1398
|
+
|
|
1399
|
+
# 如果Stream注册表为空,进行一次性的scan作为初始化(仅在首次运行时)
|
|
1400
|
+
if not stream_info_map:
|
|
1401
|
+
logger.warning(f"Stream registry is empty, performing one-time scan initialization...")
|
|
1402
|
+
pattern = f"{self.prefix}:QUEUE:*".encode()
|
|
1403
|
+
cursor = 0
|
|
1404
|
+
|
|
1405
|
+
while True:
|
|
1406
|
+
cursor, keys = await self.redis_client.scan(cursor, match=pattern, count=10000)
|
|
1407
|
+
|
|
1408
|
+
for key in keys:
|
|
1409
|
+
key_str = key.decode()
|
|
1410
|
+
# 移除前缀 "prefix:QUEUE:"
|
|
1411
|
+
queue_part = key_str.replace(f"{self.prefix}:QUEUE:", "")
|
|
1412
|
+
|
|
1413
|
+
# 检查是否是优先级队列(格式:queue_name:priority)
|
|
1414
|
+
parts = queue_part.split(':')
|
|
1415
|
+
if len(parts) == 2 and parts[1].isdigit():
|
|
1416
|
+
# 优先级队列
|
|
1417
|
+
queue_name = parts[0]
|
|
1418
|
+
priority = int(parts[1])
|
|
1419
|
+
queue_priority_key = f"{queue_name}:{priority}"
|
|
1420
|
+
elif ':' not in queue_part:
|
|
1421
|
+
# 普通队列(不包含冒号)
|
|
1422
|
+
queue_name = queue_part
|
|
1423
|
+
priority = 0
|
|
1424
|
+
queue_priority_key = queue_name
|
|
1425
|
+
else:
|
|
1426
|
+
# 忽略其他格式的键(如消费组等)
|
|
1427
|
+
continue
|
|
1428
|
+
|
|
1370
1429
|
if queue_name not in stream_info_map:
|
|
1371
1430
|
stream_info_map[queue_name] = []
|
|
1372
1431
|
stream_info_map[queue_name].append((key, priority))
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1432
|
+
|
|
1433
|
+
if cursor == 0:
|
|
1434
|
+
break
|
|
1435
|
+
|
|
1436
|
+
# 将发现的Stream信息添加到注册表中
|
|
1437
|
+
if stream_info_map:
|
|
1438
|
+
pipeline = self.redis_client.pipeline()
|
|
1439
|
+
for queue_name, stream_list in stream_info_map.items():
|
|
1440
|
+
for stream_key, priority in stream_list:
|
|
1441
|
+
if priority > 0:
|
|
1442
|
+
queue_priority_key = f"{queue_name}:{priority}"
|
|
1443
|
+
else:
|
|
1444
|
+
queue_priority_key = queue_name
|
|
1445
|
+
# stream_key已经是bytes类型(从scan_iter返回)
|
|
1446
|
+
if isinstance(stream_key, str):
|
|
1447
|
+
stream_key = stream_key.encode()
|
|
1448
|
+
pipeline.hset(self.stream_registry_key.encode(), queue_priority_key.encode(), stream_key)
|
|
1449
|
+
await pipeline.execute()
|
|
1450
|
+
logger.info(f"Registered {sum(len(stream_list) for stream_list in stream_info_map.values())} streams to registry during initialization")
|
|
1383
1451
|
|
|
1384
1452
|
if not stream_info_map:
|
|
1385
|
-
logger.debug("No streams found for backlog monitoring")
|
|
1453
|
+
logger.debug("No streams found in registry for backlog monitoring")
|
|
1386
1454
|
return
|
|
1387
1455
|
|
|
1388
1456
|
# 调试日志(使用debug级别避免刷屏)
|
jettask/{webui/run.py → run.py}
RENAMED
|
@@ -17,13 +17,13 @@ from pathlib import Path
|
|
|
17
17
|
project_root = Path(__file__).parent.parent.parent
|
|
18
18
|
sys.path.insert(0, str(project_root))
|
|
19
19
|
|
|
20
|
-
from jettask.
|
|
20
|
+
from jettask.webui_config import PostgreSQLConfig
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def run_fastapi_server(host: str = "0.0.0.0", port: int = 8000, with_consumer: bool = False):
|
|
24
24
|
"""运行FastAPI服务器"""
|
|
25
25
|
import uvicorn
|
|
26
|
-
from jettask.
|
|
26
|
+
from jettask.api import app
|
|
27
27
|
|
|
28
28
|
# 如果需要启动消费者
|
|
29
29
|
if with_consumer:
|
|
@@ -54,7 +54,7 @@ def run_gradio_interface(host: str = "0.0.0.0", port: int = 7860, share: bool =
|
|
|
54
54
|
print("You can start it with: python -m jettask.webui.run_webui --mode fastapi")
|
|
55
55
|
print()
|
|
56
56
|
|
|
57
|
-
from jettask.
|
|
57
|
+
from jettask.gradio_app import create_interface
|
|
58
58
|
|
|
59
59
|
app = create_interface()
|
|
60
60
|
print(f"Starting Gradio interface on http://{host}:{port}")
|
|
@@ -94,7 +94,7 @@ def run_combined_mode(host: str = "0.0.0.0", api_port: int = 8000, ui_port: int
|
|
|
94
94
|
|
|
95
95
|
def run_integrated_gradio(host: str = "0.0.0.0", port: int = 7860, share: bool = False):
|
|
96
96
|
"""运行集成的Gradio界面(直接访问数据源)"""
|
|
97
|
-
from jettask.
|
|
97
|
+
from jettask.integrated_gradio_app import create_integrated_interface
|
|
98
98
|
|
|
99
99
|
print("Starting Integrated Gradio interface (direct data access)...")
|
|
100
100
|
print(f"Access the interface at http://{host}:{port}")
|
jettask/scheduler/manager.py
CHANGED
|
@@ -26,6 +26,12 @@ class ScheduledTaskManager:
|
|
|
26
26
|
else:
|
|
27
27
|
# 从app对象获取pg_url
|
|
28
28
|
self.db_url = app_or_db_url.pg_url
|
|
29
|
+
|
|
30
|
+
# 将SQLAlchemy格式的URL转换为原生PostgreSQL URL
|
|
31
|
+
# postgresql+asyncpg:// -> postgresql://
|
|
32
|
+
if self.db_url and '+' in self.db_url:
|
|
33
|
+
self.db_url = self.db_url.split('+')[0] + self.db_url[self.db_url.index('://'):]
|
|
34
|
+
|
|
29
35
|
self.pool: Optional[asyncpg.Pool] = None
|
|
30
36
|
|
|
31
37
|
async def connect(self):
|
|
@@ -9,7 +9,7 @@ import traceback
|
|
|
9
9
|
from typing import Dict, Set
|
|
10
10
|
|
|
11
11
|
from jettask import Jettask
|
|
12
|
-
from jettask.
|
|
12
|
+
from jettask.task_center import TaskCenter
|
|
13
13
|
from jettask.scheduler.scheduler import TaskScheduler
|
|
14
14
|
from jettask.scheduler.manager import ScheduledTaskManager
|
|
15
15
|
|
|
@@ -219,7 +219,7 @@ def run_scheduler_for_namespace(namespace: str,
|
|
|
219
219
|
scheduler_instance = None
|
|
220
220
|
try:
|
|
221
221
|
# 构建命名空间特定的URL
|
|
222
|
-
task_center_url = f"{task_center_base_url}/api/namespaces/{namespace}"
|
|
222
|
+
task_center_url = f"{task_center_base_url}/api/v1/namespaces/{namespace}"
|
|
223
223
|
logger.info(f"连接到任务中心: {task_center_url}")
|
|
224
224
|
|
|
225
225
|
# 连接任务中心
|
|
@@ -11,7 +11,7 @@ import aiohttp
|
|
|
11
11
|
import traceback
|
|
12
12
|
|
|
13
13
|
from jettask import Jettask
|
|
14
|
-
from jettask.
|
|
14
|
+
from jettask.task_center import TaskCenter
|
|
15
15
|
from .scheduler import TaskScheduler
|
|
16
16
|
from .manager import ScheduledTaskManager
|
|
17
17
|
|
|
@@ -133,9 +133,9 @@ class UnifiedSchedulerManager:
|
|
|
133
133
|
try:
|
|
134
134
|
# 构建API URL
|
|
135
135
|
if self.task_center_url.endswith('/api'):
|
|
136
|
-
url = f"{self.task_center_url}/namespaces"
|
|
136
|
+
url = f"{self.task_center_url}/v1/namespaces"
|
|
137
137
|
else:
|
|
138
|
-
url = f"{self.task_center_url}/api/namespaces"
|
|
138
|
+
url = f"{self.task_center_url}/api/v1/namespaces"
|
|
139
139
|
|
|
140
140
|
async with aiohttp.ClientSession() as session:
|
|
141
141
|
async with session.get(url) as response:
|
|
@@ -223,9 +223,9 @@ class UnifiedSchedulerManager:
|
|
|
223
223
|
|
|
224
224
|
# 构建命名空间URL
|
|
225
225
|
if self.task_center_url.endswith('/api'):
|
|
226
|
-
namespace_url = f"{self.task_center_url}/namespaces/{namespace}"
|
|
226
|
+
namespace_url = f"{self.task_center_url}/v1/namespaces/{namespace}"
|
|
227
227
|
else:
|
|
228
|
-
namespace_url = f"{self.task_center_url}/api/namespaces/{namespace}"
|
|
228
|
+
namespace_url = f"{self.task_center_url}/api/v1/namespaces/{namespace}"
|
|
229
229
|
|
|
230
230
|
# 创建新进程
|
|
231
231
|
process = multiprocessing.Process(
|
|
@@ -8,7 +8,7 @@ import multiprocessing
|
|
|
8
8
|
from typing import Dict, Optional, Set
|
|
9
9
|
from jettask.core.unified_manager_base import UnifiedManagerBase
|
|
10
10
|
from jettask import Jettask
|
|
11
|
-
from jettask.
|
|
11
|
+
from jettask.task_center import TaskCenter
|
|
12
12
|
from .scheduler import TaskScheduler
|
|
13
13
|
from .manager import ScheduledTaskManager
|
|
14
14
|
|
|
@@ -67,18 +67,19 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
|
|
|
67
67
|
# 创建 Jettask 应用
|
|
68
68
|
app = Jettask(task_center=tc)
|
|
69
69
|
|
|
70
|
-
#
|
|
71
|
-
manager = ScheduledTaskManager(app
|
|
70
|
+
# 创建调度器管理器(不需要传递namespace参数)
|
|
71
|
+
manager = ScheduledTaskManager(app)
|
|
72
72
|
|
|
73
73
|
# 创建并启动调度器
|
|
74
74
|
self.scheduler_instance = TaskScheduler(
|
|
75
|
-
|
|
75
|
+
app=app,
|
|
76
|
+
db_manager=manager,
|
|
76
77
|
scan_interval=self.scan_interval,
|
|
77
78
|
batch_size=self.batch_size
|
|
78
79
|
)
|
|
79
80
|
|
|
80
81
|
# 运行调度器
|
|
81
|
-
await self.scheduler_instance.
|
|
82
|
+
await self.scheduler_instance.run()
|
|
82
83
|
|
|
83
84
|
except Exception as e:
|
|
84
85
|
logger.error(f"单命名空间调度器运行失败: {e}", exc_info=self.debug)
|
|
@@ -170,13 +171,20 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
|
|
|
170
171
|
async def run_scheduler():
|
|
171
172
|
try:
|
|
172
173
|
# 构建命名空间特定的URL
|
|
173
|
-
if '/api/
|
|
174
|
-
#
|
|
175
|
-
url = f"{task_center_url}/
|
|
176
|
-
|
|
177
|
-
#
|
|
174
|
+
if task_center_url.endswith('/api/v1/') or task_center_url.endswith('/api/v1'):
|
|
175
|
+
# 多命名空间模式,URL是 http://localhost:8001/api/v1/
|
|
176
|
+
url = f"{task_center_url.rstrip('/')}/namespaces/{namespace_name}"
|
|
177
|
+
elif '/api/v1/namespaces/' in task_center_url:
|
|
178
|
+
# 单命名空间模式的URL,需要替换命名空间
|
|
179
|
+
base_url = task_center_url.split('/api/v1/namespaces/')[0]
|
|
180
|
+
url = f"{base_url}/api/v1/namespaces/{namespace_name}"
|
|
181
|
+
elif '/api/namespaces/' in task_center_url:
|
|
182
|
+
# 兼容旧格式
|
|
178
183
|
base_url = task_center_url.split('/api/namespaces/')[0]
|
|
179
184
|
url = f"{base_url}/api/namespaces/{namespace_name}"
|
|
185
|
+
else:
|
|
186
|
+
# 如果是基础URL,添加命名空间路径
|
|
187
|
+
url = f"{task_center_url.rstrip('/')}/api/v1/namespaces/{namespace_name}"
|
|
180
188
|
|
|
181
189
|
# 创建任务中心连接
|
|
182
190
|
tc = TaskCenter(url)
|
|
@@ -189,7 +197,7 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
|
|
|
189
197
|
# 创建调度器管理器 - ScheduledTaskManager 只接受一个参数
|
|
190
198
|
manager = ScheduledTaskManager(app)
|
|
191
199
|
|
|
192
|
-
# 创建并启动调度器
|
|
200
|
+
# 创建并启动调度器
|
|
193
201
|
scheduler = TaskScheduler(
|
|
194
202
|
app=app,
|
|
195
203
|
db_manager=manager,
|
|
@@ -262,7 +270,7 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
|
|
|
262
270
|
if self.scheduler_instance:
|
|
263
271
|
# 单命名空间模式
|
|
264
272
|
logger.info("停止调度器")
|
|
265
|
-
|
|
273
|
+
self.scheduler_instance.stop() # stop()不是异步方法
|
|
266
274
|
|
|
267
275
|
# 多命名空间模式
|
|
268
276
|
logger.info("停止所有调度器进程")
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""
|
|
2
|
+
数据模型定义
|
|
3
|
+
所有的Pydantic模型集中管理
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
# 任务相关模型
|
|
7
|
+
from .task import (
|
|
8
|
+
TasksRequest,
|
|
9
|
+
TaskDetailResponse,
|
|
10
|
+
TaskInfo,
|
|
11
|
+
TaskActionRequest,
|
|
12
|
+
TaskListResponse
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
# 队列相关模型
|
|
16
|
+
from .queue import (
|
|
17
|
+
TimeRangeQuery,
|
|
18
|
+
QueueStatsResponse,
|
|
19
|
+
QueueTimelineResponse,
|
|
20
|
+
TrimQueueRequest,
|
|
21
|
+
QueueInfo,
|
|
22
|
+
QueueStats,
|
|
23
|
+
QueueActionRequest
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# 定时任务相关模型
|
|
27
|
+
from .scheduled_task import (
|
|
28
|
+
ScheduledTaskRequest,
|
|
29
|
+
ScheduledTaskResponse,
|
|
30
|
+
ScheduledTaskInfo,
|
|
31
|
+
ScheduleConfig,
|
|
32
|
+
ScheduledTaskCreate,
|
|
33
|
+
ScheduledTaskUpdate,
|
|
34
|
+
ScheduledTaskCreateRequest
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# 告警相关模型
|
|
38
|
+
from .alert import (
|
|
39
|
+
AlertRuleRequest,
|
|
40
|
+
AlertRuleCreate,
|
|
41
|
+
AlertRuleUpdate,
|
|
42
|
+
AlertRule,
|
|
43
|
+
AlertInstance,
|
|
44
|
+
AlertSummary
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# 命名空间相关模型
|
|
48
|
+
from .namespace import (
|
|
49
|
+
ConfigMode,
|
|
50
|
+
NamespaceCreate,
|
|
51
|
+
NamespaceUpdate,
|
|
52
|
+
NamespaceResponse,
|
|
53
|
+
NamespaceInfo,
|
|
54
|
+
NamespaceCreateRequest,
|
|
55
|
+
NamespaceUpdateRequest
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# 积压监控相关模型
|
|
59
|
+
from .backlog import (
|
|
60
|
+
BacklogTrendRequest,
|
|
61
|
+
BacklogSnapshot,
|
|
62
|
+
BacklogStatistics,
|
|
63
|
+
BacklogTrendResponse,
|
|
64
|
+
BacklogAlert
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# 通用模型
|
|
68
|
+
from .common import (
|
|
69
|
+
ErrorResponse,
|
|
70
|
+
FilterCondition,
|
|
71
|
+
BaseListRequest,
|
|
72
|
+
TimeRangeRequest,
|
|
73
|
+
BatchOperationRequest,
|
|
74
|
+
BatchOperationResponse,
|
|
75
|
+
PaginationResponse,
|
|
76
|
+
ListResponse,
|
|
77
|
+
HealthCheck,
|
|
78
|
+
SystemConfigUpdateRequest,
|
|
79
|
+
ApiResponse
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# 监控和分析模型
|
|
83
|
+
from .monitoring import (
|
|
84
|
+
MetricPoint,
|
|
85
|
+
TimeSeries,
|
|
86
|
+
MonitoringMetrics,
|
|
87
|
+
AnalyticsData,
|
|
88
|
+
SystemHealth,
|
|
89
|
+
DashboardOverviewRequest,
|
|
90
|
+
DashboardOverview,
|
|
91
|
+
WorkerMetrics
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
__all__ = [
|
|
95
|
+
# Task
|
|
96
|
+
'TasksRequest',
|
|
97
|
+
'TaskDetailResponse',
|
|
98
|
+
'TaskInfo',
|
|
99
|
+
'TaskActionRequest',
|
|
100
|
+
'TaskListResponse',
|
|
101
|
+
|
|
102
|
+
# Queue
|
|
103
|
+
'TimeRangeQuery',
|
|
104
|
+
'QueueStatsResponse',
|
|
105
|
+
'QueueTimelineResponse',
|
|
106
|
+
'TrimQueueRequest',
|
|
107
|
+
'QueueInfo',
|
|
108
|
+
'QueueStats',
|
|
109
|
+
'QueueActionRequest',
|
|
110
|
+
|
|
111
|
+
# Scheduled Task
|
|
112
|
+
'ScheduledTaskRequest',
|
|
113
|
+
'ScheduledTaskResponse',
|
|
114
|
+
'ScheduledTaskInfo',
|
|
115
|
+
'ScheduleConfig',
|
|
116
|
+
'ScheduledTaskCreate',
|
|
117
|
+
'ScheduledTaskUpdate',
|
|
118
|
+
'ScheduledTaskCreateRequest',
|
|
119
|
+
|
|
120
|
+
# Alert
|
|
121
|
+
'AlertRuleRequest',
|
|
122
|
+
'AlertRuleCreate',
|
|
123
|
+
'AlertRuleUpdate',
|
|
124
|
+
'AlertRule',
|
|
125
|
+
'AlertInstance',
|
|
126
|
+
'AlertSummary',
|
|
127
|
+
|
|
128
|
+
# Namespace
|
|
129
|
+
'ConfigMode',
|
|
130
|
+
'NamespaceCreate',
|
|
131
|
+
'NamespaceUpdate',
|
|
132
|
+
'NamespaceResponse',
|
|
133
|
+
'NamespaceInfo',
|
|
134
|
+
'NamespaceCreateRequest',
|
|
135
|
+
'NamespaceUpdateRequest',
|
|
136
|
+
|
|
137
|
+
# Backlog
|
|
138
|
+
'BacklogTrendRequest',
|
|
139
|
+
'BacklogSnapshot',
|
|
140
|
+
'BacklogStatistics',
|
|
141
|
+
'BacklogTrendResponse',
|
|
142
|
+
'BacklogAlert',
|
|
143
|
+
|
|
144
|
+
# Common
|
|
145
|
+
'ErrorResponse',
|
|
146
|
+
'FilterCondition',
|
|
147
|
+
'BaseListRequest',
|
|
148
|
+
'TimeRangeRequest',
|
|
149
|
+
'BatchOperationRequest',
|
|
150
|
+
'BatchOperationResponse',
|
|
151
|
+
'PaginationResponse',
|
|
152
|
+
'ListResponse',
|
|
153
|
+
'HealthCheck',
|
|
154
|
+
'SystemConfigUpdateRequest',
|
|
155
|
+
'ApiResponse',
|
|
156
|
+
|
|
157
|
+
# Monitoring
|
|
158
|
+
'MetricPoint',
|
|
159
|
+
'TimeSeries',
|
|
160
|
+
'MonitoringMetrics',
|
|
161
|
+
'AnalyticsData',
|
|
162
|
+
'SystemHealth',
|
|
163
|
+
'DashboardOverviewRequest',
|
|
164
|
+
'DashboardOverview',
|
|
165
|
+
'WorkerMetrics'
|
|
166
|
+
]
|