jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/webui/services/task_monitor_service.py

@@ -0,0 +1,222 @@
+"""
+任务监控服务
+
+提供任务相关的监控功能
+"""
+import logging
+from typing import Optional, Dict, Any, List
+from datetime import datetime
+import redis.asyncio as aioredis
+
+from .redis_monitor_service import RedisMonitorService
+
+logger = logging.getLogger(__name__)
+
+
+class TaskMonitorService:
+    """任务监控服务类"""
+
+    def __init__(self, redis_service: RedisMonitorService):
+        """
+        初始化任务监控服务
+
+        Args:
+            redis_service: Redis 监控基础服务实例
+        """
+        self.redis_service = redis_service
+
+    @property
+    def redis(self) -> aioredis.Redis:
+        """获取 Redis 客户端"""
+        return self.redis_service.redis
+
+    @property
+    def redis_prefix(self) -> str:
+        """获取 Redis 前缀"""
+        return self.redis_service.redis_prefix
+
+    async def get_task_info(self, stream_id: str, queue_name: str) -> Optional[Dict[str, Any]]:
+        """
+        获取单个任务的详细信息
+
+        Args:
+            stream_id: Stream ID
+            queue_name: 队列名称
+
+        Returns:
+            任务信息字典，如果任务不存在则返回 None
+        """
+        try:
+            prefixed_queue = self.redis_service.get_prefixed_queue_name(queue_name)
+
+            # 从 Stream 获取消息
+            messages = await self.redis.xrange(prefixed_queue, min=stream_id, max=stream_id, count=1)
+
+            if not messages:
+                logger.warning(f"Task not found in stream: {stream_id} in queue {queue_name}")
+                return None
+
+            msg_id, msg_data = messages[0]
+
+            # 检查消息是否在 pending 队列中
+            pending_entries = await self.redis.xpending_range(
+                prefixed_queue,
+                f"{self.redis_prefix}:GROUP:{queue_name}",
+                min=msg_id,
+                max=msg_id,
+                count=1
+            )
+
+            is_pending = len(pending_entries) > 0
+            consumer_name = pending_entries[0]["consumer"].decode() if is_pending else None
+            delivery_count = pending_entries[0]["times_delivered"] if is_pending else 0
+
+            # 构建任务信息
+            task_info = {
+                "stream_id": msg_id,
+                "queue": queue_name,
+                "data": msg_data,
+                "is_pending": is_pending,
+                "consumer": consumer_name,
+                "delivery_count": delivery_count,
+                "timestamp": int(msg_id.split('-')[0])
+            }
+
+            logger.debug(f"Retrieved task info for {stream_id}: pending={is_pending}, consumer={consumer_name}")
+            return task_info
+
+        except Exception as e:
+            logger.error(f"Error getting task info for {stream_id} in queue {queue_name}: {e}", exc_info=True)
+            return None
+
+    async def get_stream_info(self, queue_name: str) -> Optional[Dict[str, Any]]:
+        """
+        获取 Stream 的统计信息
+
+        Args:
+            queue_name: 队列名称
+
+        Returns:
+            Stream 信息字典
+        """
+        try:
+            prefixed_queue = self.redis_service.get_prefixed_queue_name(queue_name)
+
+            # 获取 Stream 信息
+            info = await self.redis.xinfo_stream(prefixed_queue)
+
+            stream_info = {
+                "length": info.get("length", 0),
+                "first_entry": info.get("first-entry"),
+                "last_entry": info.get("last-entry"),
+                "groups": info.get("groups", 0)
+            }
+
+            logger.debug(f"Retrieved stream info for queue {queue_name}: length={stream_info['length']}")
+            return stream_info
+
+        except Exception as e:
+            logger.error(f"Error getting stream info for queue {queue_name}: {e}", exc_info=True)
+            return None
+
+    async def get_queue_tasks(
+        self,
+        queue_name: str,
+        start: str = "-",
+        end: str = "+",
+        count: int = 100,
+        reverse: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        获取队列中的任务列表
+
+        Args:
+            queue_name: 队列名称
+            start: 起始 ID（默认 "-" 表示最小 ID）
+            end: 结束 ID（默认 "+" 表示最大 ID）
+            count: 返回的任务数量
+            reverse: 是否反向获取（从新到旧）
+
+        Returns:
+            任务列表
+        """
+        try:
+            prefixed_queue = self.redis_service.get_prefixed_queue_name(queue_name)
+
+            # 根据 reverse 参数选择查询方向
+            if reverse:
+                messages = await self.redis.xrevrange(prefixed_queue, max=end, min=start, count=count)
+            else:
+                messages = await self.redis.xrange(prefixed_queue, min=start, max=end, count=count)
+
+            if not messages:
+                logger.debug(f"No tasks found in queue {queue_name}")
+                return []
+
+            # 获取 pending 信息（批量查询优化）
+            group_name = f"{self.redis_prefix}:GROUP:{queue_name}"
+
+            # 尝试获取所有 pending 消息的信息
+            try:
+                pending_entries = await self.redis.xpending_range(
+                    prefixed_queue,
+                    group_name,
+                    min="-",
+                    max="+",
+                    count=10000  # 获取足够多的 pending 信息
+                )
+                # 构建 pending 映射：{msg_id: {consumer, delivery_count}}
+                pending_map = {}
+                for entry in pending_entries:
+                    msg_id = entry["message_id"]
+                    pending_map[msg_id] = {
+                        "consumer": entry["consumer"].decode() if isinstance(entry["consumer"], bytes) else entry["consumer"],
+                        "delivery_count": entry["times_delivered"]
+                    }
+            except Exception as e:
+                logger.warning(f"Error getting pending info for queue {queue_name}: {e}")
+                pending_map = {}
+
+            # 构建任务列表
+            tasks = []
+            for msg_id, msg_data in messages:
+                # 检查是否在 pending 中
+                pending_info = pending_map.get(msg_id)
+                is_pending = pending_info is not None
+
+                # 解析时间戳
+                try:
+                    timestamp_ms = int(msg_id.split('-')[0])
+                except (ValueError, IndexError):
+                    timestamp_ms = 0
+
+                # 解析消息数据
+                task_data = {}
+                for key, value in msg_data.items():
+                    # Redis 返回的值可能是 bytes
+                    if isinstance(key, bytes):
+                        key = key.decode('utf-8')
+                    if isinstance(value, bytes):
+                        value = value.decode('utf-8')
+                    task_data[key] = value
+
+                # 构建任务信息
+                task = {
+                    "stream_id": msg_id,
+                    "queue": queue_name,
+                    "data": task_data,
+                    "is_pending": is_pending,
+                    "consumer": pending_info["consumer"] if is_pending else None,
+                    "delivery_count": pending_info["delivery_count"] if is_pending else 0,
+                    "timestamp": timestamp_ms,
+                    "timestamp_iso": datetime.fromtimestamp(timestamp_ms / 1000).isoformat() if timestamp_ms else None
+                }
+
+                tasks.append(task)
+
+            logger.info(f"Retrieved {len(tasks)} tasks from queue {queue_name} (reverse={reverse})")
+            return tasks
+
+        except Exception as e:
+            logger.error(f"Error getting queue tasks for {queue_name}: {e}", exc_info=True)
+            return []

jettask/webui/services/timeline_pg_service.py

@@ -0,0 +1,452 @@
+"""
+PostgreSQL 时间轴服务
+
+从 PostgreSQL 数据库获取任务时间分布数据
+"""
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Dict, List, Any, Optional
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+logger = logging.getLogger(__name__)
+
+
+class TimelinePgService:
+    """PostgreSQL 时间轴服务"""
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def parse_iso_datetime(time_str: str) -> datetime:
+        """解析ISO格式的时间字符串，确保返回 UTC 时间"""
+        if time_str.endswith('Z'):
+            # Z 表示 UTC 时间
+            dt = datetime.fromisoformat(time_str.replace('Z', '+00:00'))
+        else:
+            dt = datetime.fromisoformat(time_str)
+
+        # 如果没有时区信息，假定为 UTC
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        # 如果有时区信息，转换为 UTC
+        elif dt.tzinfo != timezone.utc:
+            dt = dt.astimezone(timezone.utc)
+
+        return dt
+
+    @staticmethod
+    def parse_interval(interval: str) -> int:
+        """解析时间间隔字符串，返回分钟数"""
+        if interval.endswith('m'):
+            return int(interval[:-1])
+        elif interval.endswith('h'):
+            return int(interval[:-1]) * 60
+        elif interval.endswith('s'):
+            return int(interval[:-1]) // 60 if int(interval[:-1]) >= 60 else 1
+        else:
+            return 5  # 默认5分钟
+
+    @staticmethod
+    def calculate_auto_interval(duration_seconds: float) -> tuple:
+        """
+        根据时间范围自动计算合适的时间间隔
+
+        Returns:
+            tuple: (interval_seconds, interval_type, interval_str)
+        """
+        if duration_seconds <= 300:  # <= 5分钟
+            return 0.5, 'millisecond', '500ms'
+        elif duration_seconds <= 900:  # <= 15分钟
+            return 1, 'second', '1s'
+        elif duration_seconds <= 1800:  # <= 30分钟
+            return 2, 'second', '2s'
+        elif duration_seconds <= 3600:  # <= 1小时
+            return 30, 'second', '30s'
+        elif duration_seconds <= 10800:  # <= 3小时
+            return 300, 'minute', '5m'
+        elif duration_seconds <= 21600:  # <= 6小时
+            return 600, 'minute', '10m'
+        elif duration_seconds <= 43200:  # <= 12小时
+            return 1800, 'minute', '30m'
+        elif duration_seconds <= 86400:  # <= 24小时
+            return 3600, 'hour', '1h'
+        elif duration_seconds <= 172800:  # <= 2天
+            return 7200, 'hour', '2h'
+        elif duration_seconds <= 604800:  # <= 7天
+            return 21600, 'hour', '6h'
+        else:  # > 7天
+            return 86400, 'hour', '24h'
+
+    @staticmethod
+    def align_to_interval(dt: datetime, interval_seconds: float) -> datetime:
+        """对齐时间到interval_seconds的整数倍"""
+        if interval_seconds >= 3600:  # 大于等于1小时
+            # 按小时对齐
+            dt = dt.replace(minute=0, second=0, microsecond=0)
+            interval_hours = int(interval_seconds // 3600)
+            aligned_hour = (dt.hour // interval_hours) * interval_hours
+            return dt.replace(hour=aligned_hour)
+        elif interval_seconds >= 60:  # 大于等于1分钟
+            # 按分钟对齐
+            dt = dt.replace(second=0, microsecond=0)
+            interval_minutes = int(interval_seconds // 60)
+            total_minutes = dt.hour * 60 + dt.minute
+            aligned_total_minutes = (total_minutes // interval_minutes) * interval_minutes
+            aligned_hour = aligned_total_minutes // 60
+            aligned_minute = aligned_total_minutes % 60
+            return dt.replace(hour=aligned_hour, minute=aligned_minute)
+        elif interval_seconds >= 1:  # 秒级别
+            # 按秒对齐
+            dt = dt.replace(microsecond=0)
+            aligned_second = int(dt.second // interval_seconds) * int(interval_seconds)
+            return dt.replace(second=aligned_second)
+        else:  # 毫秒级别
+            # 按毫秒对齐
+            total_ms = dt.microsecond / 1000  # 转换为毫秒
+            interval_ms = interval_seconds * 1000
+            aligned_ms = int(total_ms // interval_ms) * interval_ms
+            aligned_microsecond = int(aligned_ms * 1000)
+            return dt.replace(microsecond=aligned_microsecond)
+
+    async def get_single_queue_timeline(
+        self,
+        session: AsyncSession,
+        queue_name: str,
+        start_time: Optional[str] = None,
+        end_time: Optional[str] = None,
+        interval: str = "5m"
+    ) -> Dict[str, Any]:
+        """
+        获取单个队列的时间轴数据
+
+        Args:
+            session: 数据库会话
+            queue_name: 队列名称
+            start_time: 开始时间（ISO格式字符串）
+            end_time: 结束时间（ISO格式字符串）
+            interval: 时间间隔（如 "5m", "1h"）
+
+        Returns:
+            时间轴数据
+        """
+        # 解析时间范围
+        if not end_time:
+            end_dt = datetime.now(timezone.utc)
+        else:
+            end_dt = self.parse_iso_datetime(end_time)
+
+        if not start_time:
+            start_dt = end_dt - timedelta(hours=1)
+        else:
+            start_dt = self.parse_iso_datetime(start_time)
+
+        # 解析时间间隔
+        interval_minutes = self.parse_interval(interval)
+
+        try:
+            # 使用 SQLAlchemy 的原生 SQL 查询（因为复杂的时间分组）
+            query = text(f"""
+            SELECT
+                DATE_TRUNC('minute', created_at) -
+                INTERVAL '{interval_minutes} minutes' * (EXTRACT(MINUTE FROM created_at)::int % {interval_minutes}) as time_bucket,
+                COUNT(*) as count,
+                SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed_count,
+                SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count,
+                AVG(CASE WHEN status = 'completed' AND processing_time IS NOT NULL
+                    THEN processing_time ELSE NULL END) as avg_processing_time
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+
+            result = await session.execute(query, {
+                'queue_name': queue_name,
+                'start_dt': start_dt,
+                'end_dt': end_dt
+            })
+            rows = result.mappings().all()
+
+            # 构建时间轴数据
+            timeline = []
+            for row in rows:
+                timeline.append({
+                    "time": row['time_bucket'].isoformat(),
+                    "count": row['count'],
+                    "completed_count": row['completed_count'],
+                    "failed_count": row['failed_count'],
+                    "avg_processing_time": float(row['avg_processing_time']) if row['avg_processing_time'] else 0
+                })
+
+            # 填充缺失的时间点
+            filled_timeline = []
+            current_time = start_dt
+            timeline_dict = {item['time']: item for item in timeline}
+
+            while current_time < end_dt:
+                time_key = current_time.isoformat()
+                if time_key in timeline_dict:
+                    filled_timeline.append(timeline_dict[time_key])
+                else:
+                    filled_timeline.append({
+                        "time": time_key,
+                        "count": 0,
+                        "completed_count": 0,
+                        "failed_count": 0,
+                        "avg_processing_time": 0
+                    })
+                current_time += timedelta(minutes=interval_minutes)
+
+            return {
+                "timeline": filled_timeline,
+                "interval": interval,
+                "start_time": start_dt.isoformat(),
+                "end_time": end_dt.isoformat()
+            }
+
+        except Exception as e:
+            logger.error(f"Error fetching timeline from PostgreSQL: {e}")
+            return {
+                "timeline": [],
+                "interval": interval,
+                "start_time": start_dt.isoformat(),
+                "end_time": end_dt.isoformat(),
+                "error": str(e)
+            }
+
+    def _build_time_bucket_query(
+        self,
+        interval_type: str,
+        interval_seconds: float,
+        interval_minutes: float
+    ) -> text:
+        """构建时间分组查询"""
+        if interval_type == 'millisecond':
+            return text(f"""
+            SELECT
+                DATE_TRUNC('second', created_at) +
+                INTERVAL '{interval_seconds} seconds' * FLOOR(EXTRACT(MILLISECONDS FROM created_at) / ({interval_seconds} * 1000)) as time_bucket,
+                COUNT(*) as count
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+        elif interval_type == 'second':
+            return text(f"""
+            SELECT
+                DATE_TRUNC('minute', created_at) +
+                INTERVAL '{interval_seconds} seconds' * FLOOR(EXTRACT(SECOND FROM created_at) / {interval_seconds}) as time_bucket,
+                COUNT(*) as count
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+        elif interval_type == 'minute' and interval_minutes < 60:
+            return text(f"""
+            SELECT
+                DATE_TRUNC('hour', created_at) +
+                INTERVAL '{interval_minutes} minutes' * FLOOR(EXTRACT(MINUTE FROM created_at) / {interval_minutes}) as time_bucket,
+                COUNT(*) as count
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+        elif interval_minutes == 60:
+            return text("""
+            SELECT
+                DATE_TRUNC('hour', created_at) as time_bucket,
+                COUNT(*) as count
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+        else:
+            # 大于1小时的间隔
+            interval_hours = int(interval_minutes // 60)
+            return text(f"""
+            SELECT
+                DATE_TRUNC('day', created_at) +
+                INTERVAL '{interval_hours} hours' * FLOOR(EXTRACT(HOUR FROM created_at) / {interval_hours}) as time_bucket,
+                COUNT(*) as count
+            FROM tasks
+            WHERE queue_name = :queue_name
+                AND created_at >= :start_dt
+                AND created_at < :end_dt
+            GROUP BY time_bucket
+            ORDER BY time_bucket
+            """)
+
+    async def get_multiple_queues_timeline(
+        self,
+        session_factory,
+        queues: str,
+        start_time: Optional[str] = None,
+        end_time: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        获取多个队列的时间轴数据
+
+        Args:
+            session_factory: 会话工厂
+            queues: 逗号分隔的队列名称列表
+            start_time: 开始时间（ISO格式字符串）
+            end_time: 结束时间（ISO格式字符串）
+
+        Returns:
+            多队列时间轴数据
+        """
+        # 解析队列列表
+        if not queues or queues.strip() == "":
+            end_dt = datetime.now(timezone.utc) if not end_time else self.parse_iso_datetime(end_time)
+            start_dt = (end_dt - timedelta(hours=1)) if not start_time else self.parse_iso_datetime(start_time)
+
+            return {
+                "queues": [],
+                "start_time": start_dt.isoformat(),
+                "end_time": end_dt.isoformat(),
+                "interval": "5m",
+                "message": "No queues selected"
+            }
+
+        queue_list = [q.strip() for q in queues.split(',') if q.strip()][:10]  # 最多10个队列
+
+        # 解析时间范围
+        if not end_time:
+            end_dt = datetime.now(timezone.utc)
+        else:
+            end_dt = self.parse_iso_datetime(end_time)
+
+        if not start_time:
+            start_dt = end_dt - timedelta(hours=1)
+        else:
+            start_dt = self.parse_iso_datetime(start_time)
+
+        logger.info(f'start_dt={start_dt} end_dt={end_dt}')
+
+        # 根据时间范围自动计算合适的时间间隔
+        duration = (end_dt - start_dt).total_seconds()
+        interval_seconds, interval_type, interval = self.calculate_auto_interval(duration)
+        interval_minutes = interval_seconds / 60
+
+        logger.info(f"Time range: {duration}s, using interval: {interval} -> {interval_seconds} seconds, type: {interval_type}")
+
+        result = []
+
+        for queue_name in queue_list:
+            try:
+                async with session_factory() as session:
+                    # 构建查询
+                    query = self._build_time_bucket_query(interval_type, interval_seconds, interval_minutes)
+
+                    params = {
+                        'queue_name': queue_name,
+                        'start_dt': start_dt,
+                        'end_dt': end_dt
+                    }
+
+                    # 执行查询
+                    result_obj = await session.execute(query, params)
+                    rows = result_obj.mappings().all()
+                    logger.info(f'rows={rows}')
+
+                    # 构建时间轴数据
+                    timeline = []
+                    for row in rows:
+                        timeline.append({
+                            "time": row['time_bucket'].isoformat(),
+                            "count": row['count']
+                        })
+
+                    # 填充缺失的时间点
+                    timeline_data = []
+                    for item in timeline:
+                        dt = datetime.fromisoformat(item['time'])
+                        timeline_data.append((dt, item['count']))
+
+                    # 按时间排序
+                    timeline_data.sort(key=lambda x: x[0])
+
+                    # 生成完整的时间序列
+                    filled_timeline = []
+                    current_time = self.align_to_interval(start_dt, interval_seconds)
+                    timeline_index = 0
+
+                    while current_time < end_dt:
+                        # 查找是否有匹配的数据点
+                        tolerance = timedelta(seconds=interval_seconds/2)
+                        found = False
+
+                        # 从当前位置开始查找
+                        while timeline_index < len(timeline_data):
+                            data_time, count = timeline_data[timeline_index]
+
+                            # 计算时间差（秒）
+                            time_diff = abs((data_time - current_time).total_seconds())
+
+                            if time_diff < interval_seconds / 2:
+                                # 找到匹配的数据
+                                filled_timeline.append({
+                                    "time": current_time.isoformat(),
+                                    "count": count
+                                })
+                                found = True
+                                timeline_index += 1
+                                break
+                            elif data_time > current_time + tolerance:
+                                # 数据时间已经超过当前时间太多，停止查找
+                                break
+                            else:
+                                # 这个数据点太早了，继续查找下一个
+                                timeline_index += 1
+
+                        if not found:
+                            # 没有找到匹配的数据，填充0
+                            filled_timeline.append({
+                                "time": current_time.isoformat(),
+                                "count": 0
+                            })
+
+                        current_time += timedelta(seconds=interval_seconds)
+
+                    result.append({
+                        "queue": queue_name,
+                        "timeline": {
+                            "timeline": filled_timeline,
+                            "interval": interval
+                        }
+                    })
+
+            except Exception as e:
+                logger.error(f"Error fetching timeline for queue {queue_name}: {e}")
+                result.append({
+                    "queue": queue_name,
+                    "timeline": {
+                        "timeline": [],
+                        "interval": interval,
+                        "error": str(e)
+                    }
+                })
+
+        return {
+            "queues": result,
+            "start_time": start_dt.isoformat(),
+            "end_time": end_dt.isoformat(),
+            "interval": interval
+        }