jettask 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

jettask/{api/v1 → webui/api}/scheduled.py

@@ -7,7 +7,7 @@ from typing import Optional
 import logging
 
 from jettask.schemas import ScheduledTaskRequest
-from jettask.services.scheduled_task_service import ScheduledTaskService
+from jettask.webui.services.scheduled_task_service import ScheduledTaskService
 
 router = APIRouter(prefix="/scheduled", tags=["scheduled"])
 logger = logging.getLogger(__name__)

jettask/{api/v1 → webui/api}/settings.py

@@ -6,7 +6,7 @@ from fastapi import APIRouter, HTTPException
 import logging
 import traceback
 
-from jettask.services.settings_service import SettingsService
+from jettask.webui.services.settings_service import SettingsService
 
 logger = logging.getLogger(__name__)
 

jettask/{api.py → webui/app.py}

@@ -8,6 +8,7 @@ from contextlib import asynccontextmanager
 from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Query
 from fastapi.responses import HTMLResponse
 from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
 from starlette.websockets import WebSocketState
 from redis import asyncio as aioredis
 import uvicorn
@@ -17,9 +18,9 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy import select, func, and_, or_, text
 from sqlalchemy.dialects import postgresql
 
-from jettask.pg_consumer import PostgreSQLConsumer
-from jettask.webui_config import PostgreSQLConfig, RedisConfig
-from jettask.models import Base, Task, QueueStats, Worker
+from jettask.persistence import PostgreSQLConsumer
+from jettask.webui.config import PostgreSQLConfig, RedisConfig
+from jettask.persistence.models import Base, Task
 
 logger = logging.getLogger(__name__)
 
@@ -105,6 +106,7 @@ class RedisMonitor:
         self.redis_url = redis_url
         self.redis_prefix = redis_prefix
         self.redis: Optional[aioredis.Redis] = None
+        self.worker_state_manager = None  # 延迟初始化
         self.scanner_task: Optional[asyncio.Task] = None
         self.scanner_interval = 5  # 5秒扫描一次
         self.default_heartbeat_timeout = 30  # 默认30秒心跳超时
@@ -117,19 +119,26 @@ class RedisMonitor:
         self._scanner_running = False  # 标记扫描器是否正在运行
         
     async def connect(self):
-        # 使用项目中的连接池模式，优化连接参数
-        pool = aioredis.ConnectionPool.from_url(
+        # 使用统一的连接池管理
+        from jettask.utils.db_connector import get_async_redis_pool
+
+        pool = get_async_redis_pool(
             self.redis_url,
             decode_responses=True,
-            max_connections=100,  # 增加最大连接数
-            retry_on_timeout=True,
-            retry_on_error=[ConnectionError, TimeoutError],
+            max_connections=100,
+            socket_connect_timeout=5,
+            socket_timeout=10,
             socket_keepalive=True,
-            socket_connect_timeout=5,  # 减少连接超时时间
-            socket_timeout=10,  # 设置socket超时
             health_check_interval=30
         )
         self.redis = aioredis.Redis(connection_pool=pool)
+
+        # 初始化 WorkerStateManager
+        from jettask.worker.lifecycle import WorkerStateManager
+        self.worker_state_manager = WorkerStateManager(
+            redis_client=self.redis,
+            redis_prefix=self.redis_prefix
+        )
         
     async def close(self):
         # 停止扫描器任务
@@ -387,18 +396,19 @@ class RedisMonitor:
         current_time = datetime.now(timezone.utc).timestamp()
         
         # 直接扫描所有WORKER键（排除HISTORY相关的键）
-        pattern = f"{self.redis_prefix}:WORKER:*"
-        cursor = 0
-        worker_keys = []
+        # 使用 RegistryManager 替代 scan
+        from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+        worker_registry = WorkerRegistry(
+        queue_registry = QueueRegistry(
+            redis_client=None,
+            async_redis_client=self.redis,
+            redis_prefix=self.redis_prefix
+        )
         
-        while True:
-            cursor, keys = await self.redis.scan(cursor, match=pattern, count=100)
-            # 过滤掉HISTORY相关的键
-            for key in keys:
-                if ':HISTORY:' not in key:
-                    worker_keys.append(key)
-            if cursor == 0:
-                break
+        # 获取所有 worker ID
+        worker_ids = await worker_registry.get_all_workers()
+        worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
         
         # 批量获取所有worker数据
         if worker_keys:
@@ -454,18 +464,18 @@ class RedisMonitor:
         """获取队列的worker汇总统计信息"""
         try:
             # 直接扫描所有WORKER键并过滤（排除HISTORY相关的键）
-            pattern = f"{self.redis_prefix}:WORKER:*"
-            cursor = 0
-            worker_keys = []
-            
-            while True:
-                cursor, keys = await self.redis.scan(cursor, match=pattern, count=100)
-                # 过滤掉HISTORY相关的键
-                for key in keys:
-                    if ':HISTORY:' not in key:
-                        worker_keys.append(key)
-                if cursor == 0:
-                    break
+            # 使用 RegistryManager 替代 scan
+            from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+            registry = RegistryManager(
+                redis_client=None,
+                async_redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+            
+            # 获取所有 worker ID
+            worker_ids = await worker_registry.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
             
             if not worker_keys:
                 return {
@@ -619,18 +629,18 @@ class RedisMonitor:
         """获取队列的worker汇总统计信息（快速版，不包含历史）"""
         try:
             # 直接扫描所有WORKER键（排除HISTORY相关的键）
-            pattern = f"{self.redis_prefix}:WORKER:*"
-            cursor = 0
-            worker_keys = []
-            
-            while True:
-                cursor, keys = await self.redis.scan(cursor, match=pattern, count=100)
-                # 过滤掉HISTORY相关的键
-                for key in keys:
-                    if ':HISTORY:' not in key:
-                        worker_keys.append(key)
-                if cursor == 0:
-                    break
+            # 使用 RegistryManager 替代 scan
+            from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+            registry = RegistryManager(
+                redis_client=None,
+                async_redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+            
+            # 获取所有 worker ID
+            worker_ids = await worker_registry.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
             
             if not worker_keys:
                 return {
@@ -748,16 +758,18 @@ class RedisMonitor:
             # 扫描所有WORKER键（排除HISTORY相关的键）
             pattern = f"{self.redis_prefix}:WORKER:*"
             cursor = 0
-            worker_keys = []
-            
-            while True:
-                cursor, keys = await self.redis.scan(cursor, match=pattern, count=100)
-                # 过滤掉HISTORY相关的键（虽然已经不再创建，但以防万一）
-                for key in keys:
-                    if ':HISTORY:' not in key:
-                        worker_keys.append(key)
-                if cursor == 0:
-                    break
+            # 使用 RegistryManager 替代 scan
+            from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+            registry = RegistryManager(
+                redis_client=None,
+                async_redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+            
+            # 获取所有 worker ID
+            worker_ids = await worker_registry.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
             
             if not worker_keys:
                 return []
@@ -1028,27 +1040,17 @@ class RedisMonitor:
             pattern = f"{self.redis_prefix}:QUEUE:*"
             cursor = 0
             
-            # 使用pipeline批量检查类型
-            while True:
-                cursor, keys = await self.redis.scan(cursor, match=pattern, count=500)  # 增加扫描批次大小
-                
-                if keys:
-                    # 批量检查键类型
-                    pipe = self.redis.pipeline()
-                    for key in keys:
-                        pipe.type(key)
-                    
-                    types = await pipe.execute()
-                    
-                    for i, key in enumerate(keys):
-                        if types[i] == 'stream':
-                            # 提取队列名（去掉前缀）
-                            if key.startswith(f"{self.redis_prefix}:QUEUE:"):
-                                queue_name = key[len(f"{self.redis_prefix}:QUEUE:"):]
-                                queues.add(queue_name)
-                
-                if cursor == 0:
-                    break
+            # 使用 RegistryManager 替代 scan
+            from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+            registry = RegistryManager(
+                redis_client=None,
+                async_redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+            
+            # 获取所有队列
+            queues = await queue_registry.get_all_queues()
             
             # 返回排序后的队列列表并更新缓存
             result = sorted(list(queues))
@@ -1165,51 +1167,88 @@ class RedisMonitor:
         
         while self._scanner_running:
             try:
-                # 扫描所有worker
-                pattern = f"{self.redis_prefix}:WORKER:*"
-                cursor = 0
-                worker_keys = []
+                # 使用 RegistryManager 获取所有 worker，避免 SCAN
+                from jettask.worker.manager import WorkerState as WorkerRegistry
+        from jettask.messaging.registry import QueueRegistry
+                registry = RegistryManager(
+                    redis_client=None,
+                    async_redis_client=self.redis,
+                    redis_prefix=self.redis_prefix
+                )
                 
-                while True:
-                    cursor, keys = await self.redis.scan(cursor, match=pattern, count=100)
-                    # 过滤掉HISTORY相关的键
-                    for key in keys:
-                        if ':HISTORY:' not in key:
-                            worker_keys.append(key)
-                    if cursor == 0:
-                        break
+                # 获取所有 worker ID
+                worker_ids = await worker_registry.get_all_workers()
+                
+                # 构建 worker 键
+                worker_keys = []
+                for worker_id in worker_ids:
+                    worker_key = f"{self.redis_prefix}:WORKER:{worker_id}"
+                    # 过滤掉HISTORY相关的键（虽然注册表中不应该有）
+                    if ':HISTORY:' not in worker_key:
+                        worker_keys.append(worker_key)
                 
                 if worker_keys:
-                    # 批量获取worker数据
-                    pipe = self.redis.pipeline()
-                    for key in worker_keys:
-                        pipe.hgetall(key)
-                    all_workers_data = await pipe.execute()
-                    
+                    # 通过 WorkerStateManager 批量获取 worker 数据
                     current_time = time.time()
-                    
-                    # 检查每个worker的心跳
-                    for i, worker_data in enumerate(all_workers_data):
-                        if not worker_data:
-                            continue
-                        
-                        try:
-                            # 获取心跳相关信息
-                            last_heartbeat = float(worker_data.get('last_heartbeat', 0))
-                            is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
-                            heartbeat_timeout = float(worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout))
-                            consumer_id = worker_data.get('consumer_id', '')
-                            
-                            # 检查是否超时
-                            if is_alive and (current_time - last_heartbeat) > heartbeat_timeout:
-                                logger.info(f"Worker {consumer_id} 心跳超时，标记为离线")
-                                
-                                # 更新worker状态为离线
-                                worker_key = worker_keys[i]
-                                await self.redis.hset(worker_key, 'is_alive', 'false')
-                                
-                        except Exception as e:
-                            logger.error(f"检查worker心跳时出错: {e}")
+
+                    if self.worker_state_manager:
+                        # 使用 WorkerStateManager 批量获取所有 worker 信息
+                        all_workers_info = await self.worker_state_manager.get_all_workers_info(only_alive=False)
+
+                        # 检查每个worker的心跳
+                        for worker_id in worker_ids:
+                            worker_data = all_workers_info.get(worker_id)
+                            if not worker_data:
+                                continue
+
+                            try:
+                                # 获取心跳相关信息
+                                last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                                is_alive = worker_data.get('is_alive') == 'true'
+                                heartbeat_timeout = float(worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout))
+                                consumer_id = worker_data.get('consumer_id', '')
+
+                                # 检查是否超时
+                                if is_alive and (current_time - last_heartbeat) > heartbeat_timeout:
+                                    logger.info(f"Worker {consumer_id} 心跳超时，标记为离线")
+
+                                    # 通过 WorkerStateManager 更新worker状态为离线
+                                    await self.worker_state_manager.set_worker_offline(
+                                        worker_id=worker_id,
+                                        reason="heartbeat_timeout"
+                                    )
+
+                            except Exception as e:
+                                logger.error(f"检查worker心跳时出错: {e}")
+                    else:
+                        # 降级处理：直接使用 Redis
+                        pipe = self.redis.pipeline()
+                        for key in worker_keys:
+                            pipe.hgetall(key)
+                        all_workers_data = await pipe.execute()
+
+                        # 检查每个worker的心跳
+                        for i, worker_data in enumerate(all_workers_data):
+                            if not worker_data:
+                                continue
+
+                            try:
+                                # 获取心跳相关信息
+                                last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                                is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
+                                heartbeat_timeout = float(worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout))
+                                consumer_id = worker_data.get('consumer_id', '')
+
+                                # 检查是否超时
+                                if is_alive and (current_time - last_heartbeat) > heartbeat_timeout:
+                                    logger.info(f"Worker {consumer_id} 心跳超时，标记为离线")
+
+                                    # 更新worker状态为离线
+                                    worker_key = worker_keys[i]
+                                    await self.redis.hset(worker_key, 'is_alive', 'false')
+
+                            except Exception as e:
+                                logger.error(f"检查worker心跳时出错: {e}")
                 
                 # 等待下一次扫描
                 await asyncio.sleep(self.scanner_interval)
@@ -1247,40 +1286,109 @@ pg_consumer = None
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     global pg_consumer
-    
+
     # Startup
-    await monitor.connect()
-    # 启动心跳扫描器
-    await monitor.start_heartbeat_scanner()
-    
-    # 启动PostgreSQL消费者（如果配置了且显式启用）
-    if hasattr(app.state, 'pg_config') and getattr(app.state, 'enable_consumer', False):
-        redis_config = RedisConfig.from_env()
-        pg_consumer = PostgreSQLConsumer(app.state.pg_config, redis_config)
-        await pg_consumer.start()
-        logging.info("PostgreSQL consumer started")
-    else:
-        logging.info("PostgreSQL consumer disabled (use --with-consumer to enable)")
-    
+    try:
+        import os
+        # 检查是否使用Nacos配置
+        use_nacos = os.getenv('USE_NACOS', 'false').lower() == 'true'
+
+        # 初始化数据库管理器
+        from jettask.persistence.db_manager import init_db_manager
+        await init_db_manager(use_nacos=use_nacos)
+
+        # 创建数据访问实例
+        from jettask.persistence.base import JetTaskDataAccess
+        from jettask.persistence.namespace import get_namespace_data_access
+        from jettask.config.task_center import task_center_config
+
+        data_access = JetTaskDataAccess()
+        namespace_data_access = get_namespace_data_access()
+
+        # 存储在app.state中供路由使用
+        app.state.data_access = data_access
+        app.state.namespace_data_access = namespace_data_access
+
+        # 初始化JetTask数据访问
+        await data_access.initialize()
+
+        # 记录任务中心配置
+        logger.info("=" * 60)
+        logger.info("任务中心配置:")
+        logger.info(f"  配置模式: {'Nacos' if use_nacos else '环境变量'}")
+        logger.info(f"  元数据库: {task_center_config.meta_db_host}:{task_center_config.meta_db_port}/{task_center_config.meta_db_name}")
+        logger.info(f"  API服务: {task_center_config.api_host}:{task_center_config.api_port}")
+        logger.info(f"  基础URL: {task_center_config.base_url}")
+        logger.info("=" * 60)
+
+        # 连接 monitor
+        await monitor.connect()
+        # 启动心跳扫描器
+        await monitor.start_heartbeat_scanner()
+
+        # 启动PostgreSQL消费者（如果配置了且显式启用）
+        if hasattr(app.state, 'pg_config') and getattr(app.state, 'enable_consumer', False):
+            redis_config = RedisConfig.from_env()
+            pg_consumer = PostgreSQLConsumer(app.state.pg_config, redis_config)
+            await pg_consumer.start()
+            logging.info("PostgreSQL consumer started")
+        else:
+            logging.info("PostgreSQL consumer disabled (use --with-consumer to enable)")
+
+        logger.info("JetTask WebUI 启动成功")
+    except Exception as e:
+        logger.error(f"启动失败: {e}")
+        import traceback
+        traceback.print_exc()
+        raise
+
     yield
-    
+
     # Shutdown
-    await monitor.stop_heartbeat_scanner()
-    await monitor.close()
-    
-    # 停止PostgreSQL消费者
-    if pg_consumer:
-        await pg_consumer.stop()
-    
-    # 关闭SQLAlchemy引擎
-    global async_engine
-    if async_engine:
-        await async_engine.dispose()
-        async_engine = None
-        logger.info("SQLAlchemy async engine closed")
+    try:
+        # 停止心跳扫描器
+        await monitor.stop_heartbeat_scanner()
+        await monitor.close()
+
+        # 停止PostgreSQL消费者
+        if pg_consumer:
+            await pg_consumer.stop()
+
+        # 关闭数据访问
+        if hasattr(app.state, 'data_access'):
+            await app.state.data_access.close()
+
+        # 关闭数据库管理器
+        from jettask.persistence.db_manager import close_db_manager
+        await close_db_manager()
+
+        # 关闭SQLAlchemy引擎
+        global async_engine
+        if async_engine:
+            await async_engine.dispose()
+            async_engine = None
+
+        logger.info("JetTask WebUI 关闭完成")
+    except Exception as e:
+        logger.error(f"关闭时出错: {e}")
+        import traceback
+        traceback.print_exc()
 
 app = FastAPI(title="Jettask Monitor", lifespan=lifespan)
 
+# 配置 CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # 允许所有来源（生产环境应该指定具体域名）
+    allow_credentials=True,
+    allow_methods=["*"],  # 允许所有 HTTP 方法
+    allow_headers=["*"],  # 允许所有请求头
+)
+
+# 注册 API 路由
+from jettask.webui.api import api_router
+app.include_router(api_router)
+
 
 @app.get("/api/queue/{queue_name}/tasks")
 async def get_queue_tasks(

jettask/webui/namespace_manager/__init__.py

@@ -0,0 +1,10 @@
+"""
+命名空间管理器模块
+
+提供单命名空间和多命名空间的消费者管理
+"""
+
+from .multi import NamespaceConsumerProcess
+from .unified import UnifiedConsumerManager
+
+__all__ = ['NamespaceConsumerProcess', 'UnifiedConsumerManager']