jettask 0.2.23__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/webui/middleware/__init__.py

@@ -0,0 +1,6 @@
+"""
+WebUI 中间件模块
+"""
+from .namespace_middleware import NamespaceMiddleware
+
+__all__ = ['NamespaceMiddleware']

jettask/webui/middleware/namespace_middleware.py

@@ -0,0 +1,135 @@
+"""
+Namespace 中间件 - 自动注入命名空间上下文
+
+这个中间件会自动检测路由中的 {namespace} 参数，并将 NamespaceContext 注入到 request.state.ns
+这样所有路由都无需手动使用 Depends(get_namespace_context)，直接访问 request.state.ns 即可
+"""
+import logging
+import re
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from typing import Callable
+
+from jettask.utils.namespace_dep import NamespaceContext
+
+logger = logging.getLogger(__name__)
+
+
+class NamespaceMiddleware(BaseHTTPMiddleware):
+    """
+    Namespace 自动注入中间件
+
+    功能：
+    1. 自动检测路由路径中的 {namespace} 参数
+    2. 查询命名空间配置并建立数据库连接
+    3. 将 NamespaceContext 注入到 request.state.ns
+    4. 统一处理命名空间不存在等错误
+
+    使用方式：
+    ```python
+    # 在 app.py 中注册
+    app.add_middleware(NamespaceMiddleware)
+
+    # 在路由中使用
+    @router.get("/{namespace}/queues")
+    async def get_queues(request: Request):
+        ns = request.state.ns  # 已自动注入
+        redis_client = await ns.get_redis_client()
+        # ... 业务逻辑
+    ```
+    """
+
+    # 需要排除的路径前缀（这些路径不需要 namespace）
+    EXCLUDED_PATHS = [
+        '/api/v1/namespaces',  # 命名空间管理自身
+        '/api/v1/overview/',   # 根路径（健康检查等）
+        '/docs',               # API 文档
+        '/openapi.json',       # OpenAPI schema
+        '/redoc',              # ReDoc 文档
+        '/health',             # 健康检查
+    ]
+
+    async def dispatch(self, request: Request, call_next: Callable):
+        """
+        中间件处理逻辑
+
+        Args:
+            request: HTTP 请求对象
+            call_next: 下一个中间件或路由处理器
+
+        Returns:
+            HTTP 响应
+        """
+        # 1. 检查是否是排除路径
+        path = request.url.path
+
+        # 检查排除路径
+        for excluded in self.EXCLUDED_PATHS:
+            if path.startswith(excluded):
+                # 不需要 namespace，直接放行
+                return await call_next(request)
+
+        # 2. 从路径中提取 namespace 参数
+        # 匹配模式：/api/v1/xxx/{namespace}/...
+        namespace_match = re.search(r'/api/v1/[^/]+/([^/]+)', path)
+
+        if not namespace_match:
+            # 没有 namespace 参数，直接放行
+            return await call_next(request)
+
+        namespace = namespace_match.group(1)
+
+        # 3. 特殊处理：如果 namespace 实际上是其他路径段（如 "redis"），跳过
+        # 例如：/api/v1/queues/redis/monitor/{namespace}
+        if namespace in ['redis', 'tasks-v2', 'statistics']:
+            # 尝试从更后面的路径段提取 namespace
+            # 模式：/api/v1/queues/redis/monitor/{namespace}
+            namespace_match2 = re.search(r'/api/v1/[^/]+/[^/]+/[^/]+/([^/]+)', path)
+            if namespace_match2:
+                namespace = namespace_match2.group(1)
+            else:
+                # 如果还是没有，直接放行（可能这个路由不需要 namespace）
+                return await call_next(request)
+
+        # 4. 获取 namespace_data_access
+        if not hasattr(request.app.state, 'namespace_data_access'):
+            logger.error("namespace_data_access 未初始化")
+            return JSONResponse(
+                status_code=500,
+                content={"detail": "Namespace data access not initialized"}
+            )
+
+        manager = request.app.state.namespace_data_access.manager
+
+        # 5. 获取命名空间连接并注入上下文
+        try:
+            connection = await manager.get_connection(namespace)
+
+            # 创建 NamespaceContext 并注入到 request.state
+            request.state.ns = NamespaceContext(
+                namespace_name=namespace,
+                connection=connection,
+                manager=manager
+            )
+
+            logger.debug(f"已为请求 {path} 注入命名空间上下文: {namespace}")
+
+        except ValueError as e:
+            # 命名空间不存在或配置错误
+            logger.warning(f"命名空间 '{namespace}' 不存在或配置错误: {e}")
+            return JSONResponse(
+                status_code=404,
+                content={"detail": f"命名空间 '{namespace}' 不存在或配置错误"}
+            )
+        except Exception as e:
+            # 其他错误（数据库连接失败等）
+            logger.error(f"获取命名空间 '{namespace}' 连接失败: {e}", exc_info=True)
+            return JSONResponse(
+                status_code=500,
+                content={"detail": f"获取命名空间连接失败: {str(e)}"}
+            )
+
+        # 6. 调用下一个处理器
+        response = await call_next(request)
+        return response

jettask/webui/services/__init__.py

@@ -11,6 +11,144 @@ from .analytics_service import AnalyticsService
 from .settings_service import SettingsService
 from .task_service import TaskService
 
+# 监控服务
+from .redis_monitor_service import RedisMonitorService
+from .task_monitor_service import TaskMonitorService
+from .worker_monitor_service import WorkerMonitorService
+from .queue_monitor_service import QueueMonitorService
+from .heartbeat_service import HeartbeatService
+from .timeline_service import TimelineService
+from .timeline_pg_service import TimelinePgService
+
+
+class MonitorService:
+    """
+    统一的监控服务类
+
+    整合所有监控服务，提供统一的接口
+    """
+
+    def __init__(self, redis_url: str = "redis://localhost:6379", redis_prefix: str = "jettask"):
+        """
+        初始化监控服务
+
+        Args:
+            redis_url: Redis 连接 URL
+            redis_prefix: Redis 键前缀
+        """
+        # 创建基础 Redis 服务
+        self.redis_service = RedisMonitorService(redis_url, redis_prefix)
+
+        # 创建各个子服务
+        self.task_service = TaskMonitorService(self.redis_service)
+        self.worker_service = WorkerMonitorService(self.redis_service)
+        self.queue_service = QueueMonitorService(self.redis_service)
+        self.heartbeat_service = HeartbeatService(self.redis_service)
+        self.timeline_service = TimelineService(self.redis_service)
+
+    async def connect(self):
+        """连接到 Redis"""
+        await self.redis_service.connect()
+
+    async def close(self):
+        """关闭所有服务"""
+        # 停止心跳扫描器
+        await self.heartbeat_service.stop_heartbeat_scanner()
+        # 关闭 Redis 连接
+        await self.redis_service.close()
+
+    async def __aenter__(self):
+        """异步上下文管理器入口"""
+        await self.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """异步上下文管理器退出"""
+        await self.close()
+
+    # ==================== Task Monitor Methods ====================
+
+    async def get_task_info(self, stream_id: str, queue_name: str):
+        """获取单个任务的详细信息"""
+        return await self.task_service.get_task_info(stream_id, queue_name)
+
+    async def get_stream_info(self, queue_name: str):
+        """获取 Stream 的统计信息"""
+        return await self.task_service.get_stream_info(queue_name)
+
+    async def get_queue_tasks(self, queue_name: str, start: str = "-", end: str = "+", count: int = 100, reverse: bool = False):
+        """获取队列中的任务列表"""
+        return await self.task_service.get_queue_tasks(queue_name, start, end, count, reverse)
+
+    # ==================== Worker Monitor Methods ====================
+
+    async def get_worker_heartbeats(self, queue_name: str):
+        """获取指定队列的 Worker 心跳信息"""
+        return await self.worker_service.get_worker_heartbeats(queue_name)
+
+    async def get_queue_worker_summary(self, queue_name: str):
+        """获取队列的 Worker 汇总统计信息（包含历史数据）"""
+        return await self.worker_service.get_queue_worker_summary(queue_name)
+
+    async def get_queue_worker_summary_fast(self, queue_name: str):
+        """获取队列的 Worker 汇总统计信息（快速版，仅在线 Worker）"""
+        return await self.worker_service.get_queue_worker_summary_fast(queue_name)
+
+    async def get_worker_offline_history(self, limit: int = 100, start_time=None, end_time=None):
+        """获取 Worker 下线历史记录"""
+        return await self.worker_service.get_worker_offline_history(limit, start_time, end_time)
+
+    # ==================== Queue Monitor Methods ====================
+
+    async def get_all_queues(self):
+        """获取所有队列名称"""
+        return await self.queue_service.get_all_queues()
+
+    async def get_queue_stats(self, queue_name: str):
+        """获取队列统计信息（RabbitMQ 兼容格式）"""
+        return await self.queue_service.get_queue_stats(queue_name)
+
+    # ==================== Heartbeat Service Methods ====================
+
+    async def start_heartbeat_scanner(self):
+        """启动心跳扫描器"""
+        await self.heartbeat_service.start_heartbeat_scanner()
+
+    async def stop_heartbeat_scanner(self):
+        """停止心跳扫描器"""
+        await self.heartbeat_service.stop_heartbeat_scanner()
+
+    async def check_worker_heartbeat(self, worker_id: str):
+        """检查单个 Worker 的心跳状态"""
+        return await self.heartbeat_service.check_worker_heartbeat(worker_id)
+
+    async def get_heartbeat_stats(self):
+        """获取心跳监控统计信息"""
+        return await self.heartbeat_service.get_heartbeat_stats()
+
+    # ==================== Timeline Service Methods ====================
+
+    async def get_redis_timeline(self, queue_name: str, **kwargs):
+        """获取 Redis Stream 时间轴数据"""
+        return await self.timeline_service.get_redis_timeline(queue_name, **kwargs)
+
+    # ==================== Utility Methods ====================
+
+    def get_prefixed_queue_name(self, queue_name: str) -> str:
+        """为队列名称添加前缀"""
+        return self.redis_service.get_prefixed_queue_name(queue_name)
+
+    @property
+    def redis(self):
+        """获取 Redis 客户端"""
+        return self.redis_service.redis
+
+    @property
+    def redis_prefix(self) -> str:
+        """获取 Redis 前缀"""
+        return self.redis_service.redis_prefix
+
+
 __all__ = [
     'OverviewService',
     'QueueService',
@@ -19,4 +157,12 @@ __all__ = [
     'AnalyticsService',
     'SettingsService',
     'TaskService',
+    'MonitorService',
+    'RedisMonitorService',
+    'TaskMonitorService',
+    'WorkerMonitorService',
+    'QueueMonitorService',
+    'HeartbeatService',
+    'TimelineService',
+    'TimelinePgService',
 ]

jettask/webui/services/heartbeat_service.py

@@ -0,0 +1,251 @@
+"""
+心跳监控服务
+
+提供 Worker 心跳检查和自动离线标记功能
+"""
+import asyncio
+import logging
+import time
+from typing import Optional
+
+from .redis_monitor_service import RedisMonitorService
+
+logger = logging.getLogger(__name__)
+
+
+class HeartbeatService:
+    """心跳监控服务类"""
+
+    def __init__(self, redis_service: RedisMonitorService, scanner_interval: int = 5, heartbeat_timeout: int = 30):
+        """
+        初始化心跳监控服务
+
+        Args:
+            redis_service: Redis 监控基础服务实例
+            scanner_interval: 扫描间隔（秒），默认 5 秒
+            heartbeat_timeout: 心跳超时时间（秒），默认 30 秒
+        """
+        self.redis_service = redis_service
+        self.scanner_interval = scanner_interval
+        self.default_heartbeat_timeout = heartbeat_timeout
+
+        # 扫描器任务控制
+        self.scanner_task: Optional[asyncio.Task] = None
+        self._scanner_running = False
+
+    @property
+    def redis(self):
+        """获取 Redis 客户端"""
+        return self.redis_service.redis
+
+    @property
+    def redis_prefix(self) -> str:
+        """获取 Redis 前缀"""
+        return self.redis_service.redis_prefix
+
+    @property
+    def worker_state_manager(self):
+        """获取 WorkerStateManager"""
+        return self.redis_service.worker_state_manager
+
+    async def start_heartbeat_scanner(self):
+        """启动心跳扫描器"""
+        if not self._scanner_running:
+            self._scanner_running = True
+            self.scanner_task = asyncio.create_task(self._heartbeat_scanner())
+            logger.info("心跳扫描器任务已创建并启动")
+        else:
+            logger.warning("心跳扫描器已经在运行中")
+
+    async def stop_heartbeat_scanner(self):
+        """停止心跳扫描器"""
+        self._scanner_running = False
+        if self.scanner_task and not self.scanner_task.done():
+            self.scanner_task.cancel()
+            try:
+                await self.scanner_task
+            except asyncio.CancelledError:
+                logger.info("心跳扫描器已取消")
+                pass
+            logger.info("心跳扫描器已停止")
+
+    async def _heartbeat_scanner(self):
+        """
+        心跳扫描器任务，定期检查 Worker 心跳状态
+
+        定期扫描所有 Worker 的心跳信息，如果发现心跳超时的 Worker，
+        自动标记为离线状态。
+        """
+        logger.info(f"心跳扫描器启动 (扫描间隔: {self.scanner_interval}s, 超时阈值: {self.default_heartbeat_timeout}s)")
+
+        while self._scanner_running:
+            try:
+                # 使用 WorkerStateManager 获取所有 worker
+                from jettask.worker.lifecycle import WorkerStateManager
+
+                worker_manager = WorkerStateManager(
+                    redis_client=self.redis,
+                    redis_prefix=self.redis_prefix
+                )
+
+                # 获取所有 worker ID
+                worker_ids = await worker_manager.get_all_workers()
+
+                if worker_ids:
+                    current_time = time.time()
+                    logger.debug(f"检查 {len(worker_ids)} 个 Worker 的心跳状态")
+
+                    # 批量获取所有 worker 信息
+                    all_workers_info = await worker_manager.get_all_workers_info(only_alive=False)
+
+                    # 检查每个 worker 的心跳
+                    offline_count = 0
+                    for worker_id in worker_ids:
+                        worker_data = all_workers_info.get(worker_id)
+                        if not worker_data:
+                            continue
+
+                        try:
+                            # 获取心跳相关信息
+                            last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                            is_alive = worker_data.get('is_alive') == 'true'
+                            heartbeat_timeout = float(
+                                worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout)
+                            )
+                            consumer_id = worker_data.get('consumer_id', worker_id)
+
+                            # 检查是否超时
+                            heartbeat_age = current_time - last_heartbeat
+                            if is_alive and heartbeat_age > heartbeat_timeout:
+                                logger.info(
+                                    f"Worker {consumer_id} 心跳超时 ({heartbeat_age:.1f}s > {heartbeat_timeout}s)，标记为离线"
+                                )
+
+                                # 通过 WorkerStateManager 更新 worker 状态为离线
+                                await worker_manager.set_worker_offline(
+                                    worker_id=worker_id,
+                                    reason="heartbeat_timeout"
+                                )
+                                offline_count += 1
+
+                        except Exception as e:
+                            logger.error(f"检查 worker {worker_id} 心跳时出错: {e}", exc_info=True)
+
+                    if offline_count > 0:
+                        logger.info(f"本次扫描标记了 {offline_count} 个 Worker 为离线")
+
+                # 等待下一次扫描
+                await asyncio.sleep(self.scanner_interval)
+
+            except asyncio.CancelledError:
+                logger.info("心跳扫描器收到取消信号")
+                break
+            except Exception as e:
+                logger.error(f"心跳扫描器出错: {e}", exc_info=True)
+                await asyncio.sleep(self.scanner_interval)
+
+        logger.info("心跳扫描器已停止运行")
+
+    async def check_worker_heartbeat(self, worker_id: str) -> bool:
+        """
+        检查单个 Worker 的心跳状态
+
+        Args:
+            worker_id: Worker ID
+
+        Returns:
+            如果 Worker 在线返回 True，否则返回 False
+        """
+        try:
+            worker_key = f"{self.redis_prefix}:WORKER:{worker_id}"
+            worker_data = await self.redis.hgetall(worker_key)
+
+            if not worker_data:
+                logger.warning(f"Worker {worker_id} 不存在")
+                return False
+
+            last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+            is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
+            heartbeat_timeout = float(worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout))
+
+            current_time = time.time()
+            heartbeat_age = current_time - last_heartbeat
+
+            # Worker 标记为在线且心跳未超时
+            return is_alive and heartbeat_age <= heartbeat_timeout
+
+        except Exception as e:
+            logger.error(f"检查 worker {worker_id} 心跳状态时出错: {e}", exc_info=True)
+            return False
+
+    async def get_heartbeat_stats(self) -> dict:
+        """
+        获取心跳监控统计信息
+
+        Returns:
+            包含心跳统计信息的字典
+        """
+        try:
+            from jettask.worker.lifecycle import WorkerStateManager
+
+            worker_manager = WorkerStateManager(
+                redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+
+            # 获取所有 worker
+            worker_ids = await worker_manager.get_all_workers()
+
+            if not worker_ids:
+                return {
+                    'total_workers': 0,
+                    'online_workers': 0,
+                    'timeout_workers': 0,
+                    'offline_workers': 0
+                }
+
+            # 获取所有 worker 信息
+            all_workers_info = await worker_manager.get_all_workers_info(only_alive=False)
+
+            current_time = time.time()
+            online_count = 0
+            timeout_count = 0
+            offline_count = 0
+
+            for worker_id in worker_ids:
+                worker_data = all_workers_info.get(worker_id)
+                if not worker_data:
+                    continue
+
+                last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                is_alive = worker_data.get('is_alive') == 'true'
+                heartbeat_timeout = float(worker_data.get('heartbeat_timeout', self.default_heartbeat_timeout))
+
+                heartbeat_age = current_time - last_heartbeat
+
+                if not is_alive:
+                    offline_count += 1
+                elif heartbeat_age > heartbeat_timeout:
+                    timeout_count += 1
+                else:
+                    online_count += 1
+
+            return {
+                'total_workers': len(worker_ids),
+                'online_workers': online_count,
+                'timeout_workers': timeout_count,
+                'offline_workers': offline_count,
+                'scanner_running': self._scanner_running,
+                'scanner_interval': self.scanner_interval,
+                'heartbeat_timeout': self.default_heartbeat_timeout
+            }
+
+        except Exception as e:
+            logger.error(f"获取心跳统计信息时出错: {e}", exc_info=True)
+            return {
+                'total_workers': 0,
+                'online_workers': 0,
+                'timeout_workers': 0,
+                'offline_workers': 0,
+                'error': str(e)
+            }