jettask 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

jettask/webui/backend/core/database.py

@@ -0,0 +1,200 @@
+"""
+Database connection management and utilities
+"""
+import asyncio
+from contextlib import asynccontextmanager
+from typing import Optional, Dict, Any, AsyncGenerator
+import asyncpg
+import redis.asyncio as redis
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
+from sqlalchemy.orm import DeclarativeBase
+import logging
+
+from .exceptions import DatabaseConnectionError
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseManager:
+    """数据库连接管理器"""
+    
+    def __init__(self):
+        self._pg_pools: Dict[str, asyncpg.Pool] = {}
+        self._redis_pools: Dict[str, redis.ConnectionPool] = {}
+        self._sqlalchemy_engines: Dict[str, Any] = {}
+        self._session_makers: Dict[str, async_sessionmaker] = {}
+    
+    async def get_pg_pool(self, connection_string: str, pool_name: str = "default") -> asyncpg.Pool:
+        """获取PostgreSQL连接池"""
+        if pool_name not in self._pg_pools:
+            try:
+                # 解析连接字符串
+                if connection_string.startswith('postgresql://'):
+                    connection_string = connection_string.replace('postgresql://', '')
+                elif connection_string.startswith('postgresql+asyncpg://'):
+                    connection_string = connection_string.replace('postgresql+asyncpg://', '')
+                
+                # 分离认证和主机信息
+                auth, host_info = connection_string.split('@')
+                user, password = auth.split(':')
+                host_port, database = host_info.split('/')
+                host, port = host_port.split(':')
+                
+                pool = await asyncpg.create_pool(
+                    host=host,
+                    port=int(port),
+                    user=user,
+                    password=password,
+                    database=database,
+                    min_size=2,
+                    max_size=10,
+                    command_timeout=30
+                )
+                
+                self._pg_pools[pool_name] = pool
+                logger.info(f"PostgreSQL连接池已创建: {pool_name}")
+                
+            except Exception as e:
+                logger.error(f"创建PostgreSQL连接池失败: {e}")
+                raise DatabaseConnectionError(f"PostgreSQL connection failed: {e}")
+        
+        return self._pg_pools[pool_name]
+    
+    async def get_redis_client(self, connection_string: str, pool_name: str = "default") -> redis.Redis:
+        """获取Redis客户端"""
+        if pool_name not in self._redis_pools:
+            try:
+                pool = redis.ConnectionPool.from_url(
+                    connection_string,
+                    max_connections=20,
+                    decode_responses=False  # 保持原始字节格式
+                )
+                self._redis_pools[pool_name] = pool
+                logger.info(f"Redis连接池已创建: {pool_name}")
+                
+            except Exception as e:
+                logger.error(f"创建Redis连接池失败: {e}")
+                raise DatabaseConnectionError(f"Redis connection failed: {e}")
+        
+        return redis.Redis(connection_pool=self._redis_pools[pool_name])
+    
+    def get_sqlalchemy_session_maker(
+        self, 
+        connection_string: str, 
+        pool_name: str = "default"
+    ) -> async_sessionmaker[AsyncSession]:
+        """获取SQLAlchemy会话工厂"""
+        if pool_name not in self._session_makers:
+            try:
+                # 确保使用asyncpg驱动
+                if connection_string.startswith('postgresql://'):
+                    connection_string = connection_string.replace('postgresql://', 'postgresql+asyncpg://')
+                
+                engine = create_async_engine(
+                    connection_string,
+                    pool_size=5,
+                    max_overflow=10,
+                    pool_timeout=30,
+                    pool_recycle=3600,
+                    echo=False  # 生产环境关闭SQL日志
+                )
+                
+                session_maker = async_sessionmaker(
+                    engine,
+                    class_=AsyncSession,
+                    expire_on_commit=False
+                )
+                
+                self._sqlalchemy_engines[pool_name] = engine
+                self._session_makers[pool_name] = session_maker
+                logger.info(f"SQLAlchemy会话工厂已创建: {pool_name}")
+                
+            except Exception as e:
+                logger.error(f"创建SQLAlchemy会话工厂失败: {e}")
+                raise DatabaseConnectionError(f"SQLAlchemy connection failed: {e}")
+        
+        return self._session_makers[pool_name]
+    
+    @asynccontextmanager
+    async def get_pg_connection(self, pool_name: str = "default") -> AsyncGenerator[asyncpg.Connection, None]:
+        """获取PostgreSQL连接上下文管理器"""
+        if pool_name not in self._pg_pools:
+            raise DatabaseConnectionError(f"PostgreSQL pool '{pool_name}' not found")
+        
+        pool = self._pg_pools[pool_name]
+        async with pool.acquire() as conn:
+            yield conn
+    
+    @asynccontextmanager
+    async def get_sqlalchemy_session(self, pool_name: str = "default") -> AsyncGenerator[AsyncSession, None]:
+        """获取SQLAlchemy会话上下文管理器"""
+        if pool_name not in self._session_makers:
+            raise DatabaseConnectionError(f"SQLAlchemy session maker '{pool_name}' not found")
+        
+        session_maker = self._session_makers[pool_name]
+        async with session_maker() as session:
+            try:
+                yield session
+                await session.commit()
+            except Exception:
+                await session.rollback()
+                raise
+            finally:
+                await session.close()
+    
+    async def close_all(self):
+        """关闭所有连接"""
+        # 关闭PostgreSQL连接池
+        for name, pool in self._pg_pools.items():
+            try:
+                await pool.close()
+                logger.info(f"PostgreSQL连接池已关闭: {name}")
+            except Exception as e:
+                logger.error(f"关闭PostgreSQL连接池失败 {name}: {e}")
+        
+        # 关闭Redis连接池
+        for name, pool in self._redis_pools.items():
+            try:
+                await pool.disconnect()
+                logger.info(f"Redis连接池已关闭: {name}")
+            except Exception as e:
+                logger.error(f"关闭Redis连接池失败 {name}: {e}")
+        
+        # 关闭SQLAlchemy引擎
+        for name, engine in self._sqlalchemy_engines.items():
+            try:
+                await engine.dispose()
+                logger.info(f"SQLAlchemy引擎已关闭: {name}")
+            except Exception as e:
+                logger.error(f"关闭SQLAlchemy引擎失败 {name}: {e}")
+        
+        # 清空缓存
+        self._pg_pools.clear()
+        self._redis_pools.clear()
+        self._sqlalchemy_engines.clear()
+        self._session_makers.clear()
+
+
+# 全局数据库管理器实例
+db_manager = DatabaseManager()
+
+
+# 数据库模型基类
+class Base(DeclarativeBase):
+    pass
+
+
+# 便捷函数
+async def get_pg_pool(connection_string: str, pool_name: str = "default") -> asyncpg.Pool:
+    """便捷函数：获取PostgreSQL连接池"""
+    return await db_manager.get_pg_pool(connection_string, pool_name)
+
+
+async def get_redis_client(connection_string: str, pool_name: str = "default") -> redis.Redis:
+    """便捷函数：获取Redis客户端"""
+    return await db_manager.get_redis_client(connection_string, pool_name)
+
+
+def get_session_maker(connection_string: str, pool_name: str = "default") -> async_sessionmaker[AsyncSession]:
+    """便捷函数：获取SQLAlchemy会话工厂"""
+    return db_manager.get_sqlalchemy_session_maker(connection_string, pool_name)

jettask/webui/backend/core/exceptions.py

@@ -0,0 +1,102 @@
+"""
+Custom exceptions for JetTask WebUI Backend
+"""
+from typing import Optional, Dict, Any
+from fastapi import HTTPException
+
+
+class JetTaskAPIException(HTTPException):
+    """Base exception for JetTask API"""
+    
+    def __init__(
+        self,
+        status_code: int,
+        detail: str,
+        error_code: Optional[str] = None,
+        extra_data: Optional[Dict[str, Any]] = None
+    ):
+        super().__init__(status_code=status_code, detail=detail)
+        self.error_code = error_code
+        self.extra_data = extra_data or {}
+
+
+class NamespaceNotFoundError(JetTaskAPIException):
+    """Namespace not found error"""
+    
+    def __init__(self, namespace: str):
+        super().__init__(
+            status_code=404,
+            detail=f"Namespace '{namespace}' not found",
+            error_code="NAMESPACE_NOT_FOUND",
+            extra_data={"namespace": namespace}
+        )
+
+
+class QueueNotFoundError(JetTaskAPIException):
+    """Queue not found error"""
+    
+    def __init__(self, queue_name: str, namespace: str = "default"):
+        super().__init__(
+            status_code=404,
+            detail=f"Queue '{queue_name}' not found in namespace '{namespace}'",
+            error_code="QUEUE_NOT_FOUND",
+            extra_data={"queue_name": queue_name, "namespace": namespace}
+        )
+
+
+class TaskNotFoundError(JetTaskAPIException):
+    """Task not found error"""
+    
+    def __init__(self, task_id: str):
+        super().__init__(
+            status_code=404,
+            detail=f"Task '{task_id}' not found",
+            error_code="TASK_NOT_FOUND",
+            extra_data={"task_id": task_id}
+        )
+
+
+class ValidationError(JetTaskAPIException):
+    """Validation error"""
+    
+    def __init__(self, message: str, field: Optional[str] = None):
+        super().__init__(
+            status_code=400,
+            detail=message,
+            error_code="VALIDATION_ERROR",
+            extra_data={"field": field} if field else {}
+        )
+
+
+class DatabaseConnectionError(JetTaskAPIException):
+    """Database connection error"""
+    
+    def __init__(self, message: str = "Database connection failed"):
+        super().__init__(
+            status_code=503,
+            detail=message,
+            error_code="DATABASE_CONNECTION_ERROR"
+        )
+
+
+class RateLimitError(JetTaskAPIException):
+    """Rate limit exceeded error"""
+    
+    def __init__(self, limit: int, window: int):
+        super().__init__(
+            status_code=429,
+            detail=f"Rate limit exceeded: {limit} requests per {window} seconds",
+            error_code="RATE_LIMIT_EXCEEDED",
+            extra_data={"limit": limit, "window": window}
+        )
+
+
+class InternalServerError(JetTaskAPIException):
+    """Internal server error"""
+    
+    def __init__(self, message: str = "Internal server error"):
+        super().__init__(
+            status_code=500,
+            detail=message,
+            error_code="INTERNAL_SERVER_ERROR"
+        )

jettask/webui/backend/models/__init__.py

@@ -0,0 +1,3 @@
+"""
+Data models for JetTask WebUI Backend
+"""

jettask/webui/backend/models/requests.py

@@ -0,0 +1,236 @@
+"""
+Request models for JetTask WebUI Backend
+"""
+from typing import List, Dict, Any, Optional, Union
+from datetime import datetime
+from pydantic import BaseModel, Field, validator
+from enum import Enum
+
+
+class TimeRange(str, Enum):
+    """时间范围枚举"""
+    FIFTEEN_MINUTES = "15m"
+    THIRTY_MINUTES = "30m"
+    ONE_HOUR = "1h"
+    THREE_HOURS = "3h"
+    SIX_HOURS = "6h"
+    TWELVE_HOURS = "12h"
+    ONE_DAY = "24h"
+    THREE_DAYS = "3d"
+    ONE_WEEK = "7d"
+    ONE_MONTH = "30d"
+
+
+class SortOrder(str, Enum):
+    """排序方向"""
+    ASC = "asc"
+    DESC = "desc"
+
+
+class FilterOperator(str, Enum):
+    """筛选操作符"""
+    EQ = "eq"  # 等于
+    NE = "ne"  # 不等于
+    GT = "gt"  # 大于
+    GTE = "gte"  # 大于等于
+    LT = "lt"  # 小于
+    LTE = "lte"  # 小于等于
+    IN = "in"  # 包含
+    NOT_IN = "not_in"  # 不包含
+    LIKE = "like"  # 模糊匹配
+    REGEX = "regex"  # 正则表达式
+    IS_NULL = "is_null"  # 为空
+    IS_NOT_NULL = "is_not_null"  # 不为空
+
+
+class FilterCondition(BaseModel):
+    """筛选条件"""
+    field: str = Field(description="字段名")
+    operator: FilterOperator = Field(description="操作符")
+    value: Optional[Any] = Field(default=None, description="筛选值")
+    
+    @validator('value')
+    def validate_value(cls, v, values):
+        operator = values.get('operator')
+        if operator in [FilterOperator.IS_NULL, FilterOperator.IS_NOT_NULL]:
+            return None
+        if operator in [FilterOperator.IN, FilterOperator.NOT_IN] and not isinstance(v, list):
+            raise ValueError(f"Operator {operator} requires a list value")
+        return v
+
+
+class BaseListRequest(BaseModel):
+    """基础列表查询请求"""
+    page: int = Field(default=1, ge=1, description="页码")
+    page_size: int = Field(default=20, ge=1, le=100, description="每页大小")
+    sort_field: Optional[str] = Field(default=None, description="排序字段")
+    sort_order: SortOrder = Field(default=SortOrder.DESC, description="排序方向")
+    filters: List[FilterCondition] = Field(default_factory=list, description="筛选条件")
+    search: Optional[str] = Field(default=None, description="搜索关键词")
+
+
+class TimeRangeRequest(BaseModel):
+    """时间范围查询请求"""
+    start_time: Optional[datetime] = Field(default=None, description="开始时间")
+    end_time: Optional[datetime] = Field(default=None, description="结束时间")
+    time_range: Optional[TimeRange] = Field(default=None, description="预设时间范围")
+    granularity: Optional[str] = Field(default=None, description="数据粒度")
+    
+    @validator('end_time')
+    def validate_time_range(cls, v, values):
+        start_time = values.get('start_time')
+        if start_time and v and start_time >= v:
+            raise ValueError("end_time must be after start_time")
+        return v
+
+
+# 队列相关请求模型
+class QueueListRequest(BaseListRequest):
+    """队列列表查询请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    status: Optional[str] = Field(default=None, description="队列状态筛选")
+    include_stats: bool = Field(default=True, description="是否包含统计信息")
+
+
+class QueueMetricsRequest(TimeRangeRequest):
+    """队列指标查询请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    queue_name: str = Field(description="队列名称")
+    metrics: List[str] = Field(default_factory=lambda: ["pending", "processing", "completed"], description="指标类型")
+    include_consumer_groups: bool = Field(default=False, description="是否包含消费者组数据")
+
+
+class QueueActionRequest(BaseModel):
+    """队列操作请求"""
+    action: str = Field(description="操作类型")
+    parameters: Dict[str, Any] = Field(default_factory=dict, description="操作参数")
+
+
+# 任务相关请求模型
+class TaskListRequest(BaseListRequest, TimeRangeRequest):
+    """任务列表查询请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    queue_name: Optional[str] = Field(default=None, description="队列名称")
+    status: Optional[str] = Field(default=None, description="任务状态")
+    consumer_group: Optional[str] = Field(default=None, description="消费者组")
+    worker_id: Optional[str] = Field(default=None, description="工作者ID")
+    task_name: Optional[str] = Field(default=None, description="任务名称")
+
+
+class TaskActionRequest(BaseModel):
+    """任务操作请求"""
+    task_ids: List[str] = Field(description="任务ID列表")
+    action: str = Field(description="操作类型")
+    parameters: Dict[str, Any] = Field(default_factory=dict, description="操作参数")
+
+
+# 监控相关请求模型
+class MonitoringRequest(TimeRangeRequest):
+    """监控数据请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    queues: Optional[List[str]] = Field(default=None, description="队列列表")
+    metrics: List[str] = Field(default_factory=list, description="指标类型")
+    include_groups: bool = Field(default=False, description="是否包含消费者组")
+
+
+class BacklogTrendRequest(TimeRangeRequest):
+    """队列积压趋势请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    queues: Optional[List[str]] = Field(default=None, description="队列列表")
+    include_groups: bool = Field(default=False, description="是否包含消费者组级别数据")
+
+
+# 分析相关请求模型
+class AnalyticsRequest(TimeRangeRequest):
+    """分析查询请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    analysis_type: str = Field(description="分析类型")
+    dimensions: List[str] = Field(default_factory=list, description="分析维度")
+    metrics: List[str] = Field(default_factory=list, description="分析指标")
+    filters: List[FilterCondition] = Field(default_factory=list, description="筛选条件")
+
+
+# 定时任务相关请求模型
+class ScheduleConfig(BaseModel):
+    """调度配置"""
+    cron_expression: Optional[str] = Field(default=None, description="Cron表达式")
+    interval_seconds: Optional[int] = Field(default=None, description="间隔秒数")
+    interval_minutes: Optional[int] = Field(default=None, description="间隔分钟数")
+    interval_hours: Optional[int] = Field(default=None, description="间隔小时数")
+    start_date: Optional[datetime] = Field(default=None, description="开始时间")
+    end_date: Optional[datetime] = Field(default=None, description="结束时间")
+    max_runs: Optional[int] = Field(default=None, description="最大运行次数")
+    
+    @validator('interval_seconds')
+    def validate_interval_seconds(cls, v):
+        if v is not None and v < 1:
+            raise ValueError("interval_seconds must be at least 1")
+        return v
+
+
+class ScheduledTaskCreateRequest(BaseModel):
+    """创建定时任务请求"""
+    name: str = Field(description="任务名称")
+    namespace: str = Field(default="default", description="命名空间")
+    queue_name: str = Field(description="队列名称")
+    task_name: str = Field(description="任务函数名")
+    task_data: Dict[str, Any] = Field(default_factory=dict, description="任务数据")
+    schedule_type: str = Field(description="调度类型")
+    schedule_config: ScheduleConfig = Field(description="调度配置")
+    is_active: bool = Field(default=True, description="是否启用")
+    description: Optional[str] = Field(default=None, description="任务描述")
+    max_retry: int = Field(default=3, ge=0, description="最大重试次数")
+    timeout: Optional[int] = Field(default=None, ge=1, description="超时时间(秒)")
+
+
+class ScheduledTaskUpdateRequest(ScheduledTaskCreateRequest):
+    """更新定时任务请求"""
+    pass
+
+
+class ScheduledTaskListRequest(BaseListRequest):
+    """定时任务列表查询请求"""
+    namespace: str = Field(default="default", description="命名空间")
+    is_active: Optional[bool] = Field(default=None, description="是否启用")
+    schedule_type: Optional[str] = Field(default=None, description="调度类型")
+
+
+# 命名空间相关请求模型
+class NamespaceCreateRequest(BaseModel):
+    """创建命名空间请求"""
+    name: str = Field(description="命名空间名称", pattern="^[a-zA-Z0-9_-]+$")
+    display_name: str = Field(description="显示名称")
+    description: Optional[str] = Field(default=None, description="描述")
+    redis_url: str = Field(description="Redis连接URL")
+    pg_url: Optional[str] = Field(default=None, description="PostgreSQL连接URL")
+    redis_prefix: str = Field(default="jettask", description="Redis键前缀")
+
+
+class NamespaceUpdateRequest(BaseModel):
+    """更新命名空间请求"""
+    display_name: Optional[str] = Field(default=None, description="显示名称")
+    description: Optional[str] = Field(default=None, description="描述")
+    redis_url: Optional[str] = Field(default=None, description="Redis连接URL")
+    pg_url: Optional[str] = Field(default=None, description="PostgreSQL连接URL")
+    is_active: Optional[bool] = Field(default=None, description="是否启用")
+
+
+class NamespaceListRequest(BaseListRequest):
+    """命名空间列表查询请求"""
+    is_active: Optional[bool] = Field(default=None, description="是否启用")
+
+
+# 系统管理相关请求模型
+class SystemConfigUpdateRequest(BaseModel):
+    """系统配置更新请求"""
+    config_key: str = Field(description="配置键")
+    config_value: Any = Field(description="配置值")
+    description: Optional[str] = Field(default=None, description="配置描述")
+
+
+class BatchOperationRequest(BaseModel):
+    """批量操作请求"""
+    operation: str = Field(description="操作类型")
+    target_type: str = Field(description="目标类型")
+    target_ids: List[str] = Field(description="目标ID列表")
+    parameters: Dict[str, Any] = Field(default_factory=dict, description="操作参数")