jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/webui/services/timeline_service.py

@@ -0,0 +1,189 @@
+"""
+时间轴服务
+
+提供队列任务的时间分布分析功能
+"""
+import logging
+from typing import Dict, Any, List, Optional
+from datetime import datetime, timedelta, timezone
+
+from .redis_monitor_service import RedisMonitorService
+
+logger = logging.getLogger(__name__)
+
+
+class TimelineService:
+    """时间轴服务类"""
+
+    def __init__(self, redis_service: RedisMonitorService):
+        """
+        初始化时间轴服务
+
+        Args:
+            redis_service: Redis 监控基础服务实例
+        """
+        self.redis_service = redis_service
+
+    @property
+    def redis(self):
+        """获取 Redis 客户端"""
+        return self.redis_service.redis
+
+    @property
+    def redis_prefix(self) -> str:
+        """获取 Redis 前缀"""
+        return self.redis_service.redis_prefix
+
+    async def get_redis_timeline(
+        self,
+        queue_name: str,
+        interval: str = "1m",
+        duration: str = "1h",
+        start_time: Optional[str] = None,
+        end_time: Optional[str] = None,
+        context: str = "detail"
+    ) -> Dict[str, Any]:
+        """
+        从 Redis Stream 获取队列任务的时间分布
+
+        Args:
+            queue_name: 队列名称
+            interval: 时间间隔 (如 '1m', '5m', '1h')
+            duration: 持续时间 (如 '1h', '24h')
+            start_time: 开始时间（可选）
+            end_time: 结束时间（可选）
+            context: 上下文 ('overview' 或 'detail')
+
+        Returns:
+            时间轴数据
+        """
+        try:
+            # 解析时间间隔和持续时间
+            interval_seconds = self._parse_time_duration(interval)
+
+            # 根据上下文设置不同的数据限制
+            if context == "overview":
+                # 首页概览：固定获取最近1小时的所有数据
+                duration_seconds = 3600  # 1小时
+                now = int(datetime.now(timezone.utc).timestamp() * 1000)
+                start = now - duration_seconds * 1000
+                min_id = f"{start}-0"
+                max_id = "+"
+                max_count = 100000  # 首页概览获取所有数据
+            else:
+                # 队列详情页：根据参数获取，但限制最多10000条
+                if start_time and end_time:
+                    # 使用提供的时间范围
+                    min_id = start_time
+                    max_id = end_time if end_time != '+' else '+'
+                else:
+                    # 使用duration参数计算时间范围
+                    duration_seconds = self._parse_time_duration(duration)
+                    now = int(datetime.now(timezone.utc).timestamp() * 1000)
+                    start = now - duration_seconds * 1000
+                    min_id = f"{start}-0"
+                    max_id = "+"
+                max_count = 10000  # 详情页限制10000条
+
+            # 获取指定时间范围内的消息
+            prefixed_queue_name = self.redis_service.get_prefixed_queue_name(queue_name)
+            messages = await self.redis.xrange(
+                prefixed_queue_name,
+                min=min_id,
+                max=max_id,
+                count=max_count
+            )
+
+            # 按时间间隔统计任务数量
+            buckets = {}
+            bucket_size = interval_seconds * 1000  # 转换为毫秒
+
+            # 计算实际的时间范围用于生成时间轴
+            if start_time and end_time:
+                # 从参数中解析时间范围
+                if start_time != '-':
+                    actual_start = int(start_time.split('-')[0])
+                else:
+                    actual_start = int(datetime.now(timezone.utc).timestamp() * 1000) - 86400000
+
+                if end_time != '+':
+                    actual_end = int(end_time.split('-')[0])
+                else:
+                    actual_end = int(datetime.now(timezone.utc).timestamp() * 1000)
+            else:
+                # 使用duration参数计算的时间范围
+                actual_start = start
+                actual_end = now
+
+            for msg_id, _ in messages:
+                # 从消息ID提取时间戳
+                timestamp = int(msg_id.split('-')[0])
+                bucket_key = (timestamp // bucket_size) * bucket_size
+                buckets[bucket_key] = buckets.get(bucket_key, 0) + 1
+
+            # 转换为时间序列数据
+            timeline_data = []
+            current_bucket = (actual_start // bucket_size) * bucket_size
+
+            while current_bucket <= actual_end:
+                timeline_data.append({
+                    "timestamp": current_bucket,
+                    "count": buckets.get(current_bucket, 0)
+                })
+                current_bucket += bucket_size
+
+            # 计算实际任务总数
+            total_tasks = len(messages)
+
+            # 检查是否达到数据限制
+            has_more = False
+            if context == "detail" and total_tasks >= max_count:
+                has_more = True
+
+            logger.info(f"Redis 时间轴: 队列={queue_name}, 任务数={total_tasks}, 数据点={len(timeline_data)}")
+
+            return {
+                "timeline": timeline_data,
+                "interval": interval,
+                "duration": duration,
+                "start": actual_start,
+                "end": actual_end,
+                "total_tasks": total_tasks,
+                "message_count": len(messages),
+                "has_more": has_more,
+                "limit": max_count if context == "detail" else None,
+                "source": "redis"
+            }
+
+        except Exception as e:
+            logger.error(f"获取 Redis 时间轴失败: 队列={queue_name}, 错误={e}", exc_info=True)
+            return {
+                "timeline": [],
+                "error": str(e),
+                "source": "redis"
+            }
+
+    def _parse_time_duration(self, duration_str: str) -> int:
+        """
+        解析时间字符串为秒数
+
+        Args:
+            duration_str: 时间字符串 (如 '1h', '10m', '30s')
+
+        Returns:
+            秒数
+        """
+        units = {
+            's': 1,
+            'm': 60,
+            'h': 3600,
+            'd': 86400
+        }
+
+        if duration_str[-1] in units:
+            value = int(duration_str[:-1])
+            unit = duration_str[-1]
+            return value * units[unit]
+
+        # 默认为秒
+        return int(duration_str)

jettask/webui/services/worker_monitor_service.py

@@ -0,0 +1,467 @@
+"""
+Worker 监控服务
+
+提供 Worker 相关的监控功能
+"""
+import logging
+from typing import Dict, Any, List, Optional
+from datetime import datetime, timezone
+import time
+
+from .redis_monitor_service import RedisMonitorService
+
+logger = logging.getLogger(__name__)
+
+
+class WorkerMonitorService:
+    """Worker 监控服务类"""
+
+    def __init__(self, redis_service: RedisMonitorService):
+        """
+        初始化 Worker 监控服务
+
+        Args:
+            redis_service: Redis 监控基础服务实例
+        """
+        self.redis_service = redis_service
+
+    @property
+    def redis(self):
+        """获取 Redis 客户端"""
+        return self.redis_service.redis
+
+    @property
+    def redis_prefix(self) -> str:
+        """获取 Redis 前缀"""
+        return self.redis_service.redis_prefix
+
+    async def get_worker_heartbeats(self, queue_name: str) -> List[Dict[str, Any]]:
+        """
+        获取指定队列的 Worker 心跳信息
+
+        Args:
+            queue_name: 队列名称
+
+        Returns:
+            Worker 心跳信息列表
+        """
+        worker_list = []
+        current_time = datetime.now(timezone.utc).timestamp()
+
+        try:
+            # 使用 WorkerStateManager 获取所有 worker
+            from jettask.worker.lifecycle import WorkerStateManager
+
+            worker_manager = WorkerStateManager(
+                redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+
+            # 获取所有 worker ID
+            worker_ids = await worker_manager.get_all_workers()
+            if not worker_ids:
+                logger.debug(f"No workers found for queue {queue_name}")
+                return []
+
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
+
+            # 批量获取所有 worker 数据
+            pipe = self.redis.pipeline()
+            for key in worker_keys:
+                pipe.hgetall(key)
+            all_workers_data = await pipe.execute()
+
+            # 处理每个 worker
+            for i, worker_data in enumerate(all_workers_data):
+                if not worker_data:
+                    continue
+
+                # 检查 worker 是否属于指定队列
+                worker_queues = worker_data.get('queues', '')
+                if queue_name not in worker_queues.split(','):
+                    continue
+
+                worker_id = worker_keys[i].split(':')[-1]
+                last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
+                consumer_id = worker_data.get('consumer_id', worker_id)
+
+                # 构建显示数据
+                display_data = {
+                    'consumer_id': consumer_id,
+                    'consumer_name': f"{consumer_id}-{queue_name}",
+                    'host': worker_data.get('host', 'unknown'),
+                    'pid': int(worker_data.get('pid', 0)),
+                    'queue': queue_name,
+                    'last_heartbeat': last_heartbeat,
+                    'last_heartbeat_time': datetime.fromtimestamp(last_heartbeat).isoformat(),
+                    'seconds_ago': int(current_time - last_heartbeat),
+                    'is_alive': is_alive,
+                    # 队列特定的统计信息
+                    'success_count': int(worker_data.get(f'{queue_name}:success_count', 0)),
+                    'failed_count': int(worker_data.get(f'{queue_name}:failed_count', 0)),
+                    'total_count': int(worker_data.get(f'{queue_name}:total_count', 0)),
+                    'running_tasks': int(worker_data.get(f'{queue_name}:running_tasks', 0)),
+                    'avg_processing_time': float(worker_data.get(f'{queue_name}:avg_processing_time', 0.0)),
+                    'avg_latency_time': float(worker_data.get(f'{queue_name}:avg_latency_time', 0.0))
+                }
+
+                # 如果离线时间存在，添加离线时间信息
+                if 'offline_time' in worker_data:
+                    display_data['offline_time'] = float(worker_data['offline_time'])
+                    display_data['offline_time_formatted'] = datetime.fromtimestamp(
+                        float(worker_data['offline_time'])
+                    ).isoformat()
+
+                worker_list.append(display_data)
+
+            logger.info(f"Retrieved {len(worker_list)} workers for queue {queue_name}")
+            return worker_list
+
+        except Exception as e:
+            logger.error(f"Error getting worker heartbeats for queue {queue_name}: {e}", exc_info=True)
+            return []
+
+    async def get_queue_worker_summary(self, queue_name: str) -> Dict[str, Any]:
+        """
+        获取队列的 Worker 汇总统计信息（包含历史数据）
+
+        Args:
+            queue_name: 队列名称
+
+        Returns:
+            Worker 汇总统计字典
+        """
+        try:
+            # 使用 WorkerStateManager
+            from jettask.worker.lifecycle import WorkerStateManager
+
+            worker_manager = WorkerStateManager(
+                redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+
+            # 获取所有 worker ID
+            worker_ids = await worker_manager.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
+
+            if not worker_keys:
+                return self._empty_summary()
+
+            # 批量获取 worker 数据
+            pipe = self.redis.pipeline()
+            for key in worker_keys:
+                pipe.hgetall(key)
+            all_workers_data = await pipe.execute()
+
+            # 过滤属于该队列的 worker
+            queue_workers_data = []
+            for worker_data in all_workers_data:
+                if worker_data and queue_name in worker_data.get('queues', '').split(','):
+                    queue_workers_data.append(worker_data)
+
+            # 汇总统计
+            stats = self._calculate_worker_stats(queue_workers_data, queue_name, include_history=True)
+            stats['history_included'] = True
+
+            logger.debug(f"Worker summary for queue {queue_name}: {stats}")
+            return stats
+
+        except Exception as e:
+            logger.error(f"Error getting queue worker summary for {queue_name}: {e}", exc_info=True)
+            return self._empty_summary()
+
+    async def get_queue_worker_summary_fast(self, queue_name: str) -> Dict[str, Any]:
+        """
+        获取队列的 Worker 汇总统计信息（快速版，仅在线 Worker）
+
+        Args:
+            queue_name: 队列名称
+
+        Returns:
+            Worker 汇总统计字典
+        """
+        try:
+            # 使用 WorkerStateManager
+            from jettask.worker.lifecycle import WorkerStateManager
+
+            worker_manager = WorkerStateManager(
+                redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+
+            # 获取所有 worker ID
+            worker_ids = await worker_manager.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
+
+            if not worker_keys:
+                return self._empty_summary()
+
+            # 批量获取 worker 数据
+            pipe = self.redis.pipeline()
+            for worker_key in worker_keys:
+                pipe.hgetall(worker_key)
+
+            all_workers_data = await pipe.execute()
+
+            # 过滤属于该队列的 worker
+            worker_data_list = []
+            for worker_data in all_workers_data:
+                if worker_data and queue_name in worker_data.get('queues', '').split(','):
+                    worker_data_list.append(worker_data)
+
+            # 汇总统计（仅在线 Worker）
+            stats = self._calculate_worker_stats(worker_data_list, queue_name, include_history=False)
+
+            logger.debug(f"Fast worker summary for queue {queue_name}: {stats}")
+            return stats
+
+        except Exception as e:
+            logger.error(f"Error getting fast queue worker summary for {queue_name}: {e}", exc_info=True)
+            return self._empty_summary()
+
+    async def get_worker_offline_history(
+        self,
+        limit: int = 100,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        获取 Worker 下线历史记录
+
+        Args:
+            limit: 返回记录数量限制
+            start_time: 开始时间戳（可选）
+            end_time: 结束时间戳（可选）
+
+        Returns:
+            离线历史记录列表
+        """
+        try:
+            # 使用 WorkerStateManager
+            from jettask.worker.lifecycle import WorkerStateManager
+
+            worker_manager = WorkerStateManager(
+                redis_client=self.redis,
+                redis_prefix=self.redis_prefix
+            )
+
+            # 获取所有 worker ID
+            worker_ids = await worker_manager.get_all_workers()
+            worker_keys = [f"{self.redis_prefix}:WORKER:{wid}" for wid in worker_ids]
+
+            if not worker_keys:
+                return []
+
+            # 批量获取所有 worker 数据
+            pipe = self.redis.pipeline()
+            for key in worker_keys:
+                pipe.hgetall(key)
+            all_workers_data = await pipe.execute()
+
+            # 收集离线的 worker
+            offline_workers = []
+
+            for worker_data in all_workers_data:
+                if not worker_data:
+                    continue
+
+                # 检查是否离线
+                is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
+                if not is_alive and 'offline_time' in worker_data:
+                    offline_time = float(worker_data.get('offline_time', 0))
+
+                    # 时间范围过滤
+                    if start_time and offline_time < start_time:
+                        continue
+                    if end_time and offline_time > end_time:
+                        continue
+
+                    # 构建离线记录
+                    record = self._build_offline_record(worker_data, offline_time)
+                    offline_workers.append((offline_time, record))
+
+            # 按离线时间倒序排序
+            offline_workers.sort(key=lambda x: x[0], reverse=True)
+
+            # 返回指定数量的记录
+            result = [record for _, record in offline_workers[:limit]]
+            logger.info(f"Retrieved {len(result)} offline worker records")
+            return result
+
+        except Exception as e:
+            logger.error(f"Error getting worker offline history: {e}", exc_info=True)
+            return []
+
+    def _empty_summary(self) -> Dict[str, Any]:
+        """返回空的汇总统计"""
+        return {
+            'total_workers': 0,
+            'online_workers': 0,
+            'offline_workers': 0,
+            'total_success_count': 0,
+            'total_failed_count': 0,
+            'total_count': 0,
+            'total_running_tasks': 0,
+            'avg_processing_time': 0.0,
+            'avg_latency_time': 0.0
+        }
+
+    def _calculate_worker_stats(
+        self,
+        workers_data: List[Dict[str, Any]],
+        queue_name: str,
+        include_history: bool
+    ) -> Dict[str, Any]:
+        """
+        计算 Worker 统计信息
+
+        Args:
+            workers_data: Worker 数据列表
+            queue_name: 队列名称
+            include_history: 是否包含离线 Worker 的历史数据
+
+        Returns:
+            统计信息字典
+        """
+        total_workers = len(workers_data)
+        online_workers = 0
+        offline_workers = 0
+        total_success_count = 0
+        total_failed_count = 0
+        total_count = 0
+        total_running_tasks = 0
+        total_processing_time = 0.0
+        processing_time_count = 0
+        total_latency_time = 0.0
+        latency_time_count = 0
+
+        current_time = datetime.now(timezone.utc).timestamp()
+
+        for worker_data in workers_data:
+            try:
+                last_heartbeat = float(worker_data.get('last_heartbeat', 0))
+                is_alive = worker_data.get('is_alive', 'true').lower() == 'true'
+
+                is_online = is_alive and (current_time - last_heartbeat) < 30
+
+                if is_online:
+                    online_workers += 1
+                else:
+                    offline_workers += 1
+                    # 如果不包含历史，跳过离线 Worker 的统计
+                    if not include_history:
+                        continue
+
+                # 统计数据
+                success_count = int(worker_data.get(f'{queue_name}:success_count', 0))
+                failed_count = int(worker_data.get(f'{queue_name}:failed_count', 0))
+                running_tasks = int(worker_data.get(f'{queue_name}:running_tasks', 0))
+                avg_processing_time = float(worker_data.get(f'{queue_name}:avg_processing_time', 0.0))
+                avg_latency_time = float(worker_data.get(f'{queue_name}:avg_latency_time', 0.0))
+
+                total_success_count += success_count
+                total_failed_count += failed_count
+                total_count += success_count + failed_count
+                total_running_tasks += running_tasks
+
+                if avg_processing_time > 0:
+                    total_processing_time += avg_processing_time
+                    processing_time_count += 1
+
+                if avg_latency_time > 0:
+                    total_latency_time += avg_latency_time
+                    latency_time_count += 1
+
+            except Exception as e:
+                logger.warning(f"Error processing worker stats: {e}")
+                continue
+
+        # 计算平均值
+        overall_avg_processing_time = 0.0
+        if processing_time_count > 0:
+            overall_avg_processing_time = total_processing_time / processing_time_count
+
+        overall_avg_latency_time = 0.0
+        if latency_time_count > 0:
+            overall_avg_latency_time = total_latency_time / latency_time_count
+
+        return {
+            'total_workers': total_workers,
+            'online_workers': online_workers,
+            'offline_workers': offline_workers,
+            'total_success_count': total_success_count,
+            'total_failed_count': total_failed_count,
+            'total_count': total_count,
+            'total_running_tasks': total_running_tasks,
+            'avg_processing_time': round(overall_avg_processing_time, 3),
+            'avg_latency_time': round(overall_avg_latency_time, 3)
+        }
+
+    def _build_offline_record(self, worker_data: Dict[str, Any], offline_time: float) -> Dict[str, Any]:
+        """
+        构建离线记录
+
+        Args:
+            worker_data: Worker 数据
+            offline_time: 离线时间戳
+
+        Returns:
+            离线记录字典
+        """
+        # 计算运行时长
+        online_time = float(worker_data.get('created_at', offline_time))
+        duration_seconds = int(offline_time - online_time)
+
+        # 基础记录
+        record = {
+            'consumer_id': worker_data.get('consumer_id', ''),
+            'host': worker_data.get('host', 'unknown'),
+            'pid': int(worker_data.get('pid', 0)),
+            'queues': worker_data.get('queues', ''),
+            'online_time': online_time,
+            'offline_time': offline_time,
+            'duration_seconds': duration_seconds,
+            'last_heartbeat': float(worker_data.get('last_heartbeat', 0)),
+            'shutdown_reason': worker_data.get('shutdown_reason', 'unknown'),
+            'online_time_str': datetime.fromtimestamp(online_time).isoformat(),
+            'offline_time_str': datetime.fromtimestamp(offline_time).isoformat(),
+        }
+
+        # 格式化运行时长
+        hours = duration_seconds // 3600
+        minutes = (duration_seconds % 3600) // 60
+        seconds = duration_seconds % 60
+        record['duration_str'] = f"{hours}h {minutes}m {seconds}s"
+
+        # 添加统计信息（聚合所有队列的数据）
+        queues = worker_data.get('queues', '').split(',') if worker_data.get('queues') else []
+        total_success = 0
+        total_failed = 0
+        total_count = 0
+        total_processing_time = 0.0
+
+        for queue in queues:
+            if queue.strip():
+                queue = queue.strip()
+                total_success += int(worker_data.get(f'{queue}:success_count', 0))
+                total_failed += int(worker_data.get(f'{queue}:failed_count', 0))
+                count = int(worker_data.get(f'{queue}:total_count', 0))
+                total_count += count
+
+                avg_time = float(worker_data.get(f'{queue}:avg_processing_time', 0))
+                if avg_time > 0 and count > 0:
+                    total_processing_time += avg_time * count
+
+        record['total_success_count'] = total_success
+        record['total_failed_count'] = total_failed
+        record['total_count'] = total_count
+        record['total_running_tasks'] = 0  # 离线 worker 没有运行中的任务
+
+        # 计算平均处理时间
+        if total_count > 0:
+            record['avg_processing_time'] = total_processing_time / total_count
+        else:
+            record['avg_processing_time'] = 0.0
+
+        return record

jettask/webui/utils/__init__.py

@@ -0,0 +1,11 @@
+"""
+工具函数模块
+"""
+
+from .time_utils import parse_iso_datetime, format_task_timestamps, task_obj_to_dict
+
+__all__ = [
+    'parse_iso_datetime',
+    'format_task_timestamps',
+    'task_obj_to_dict',
+]