jettask 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

jettask/scheduler/models.py

@@ -0,0 +1,200 @@
+"""
+定时任务数据模型
+"""
+import json
+from datetime import datetime, timedelta
+from typing import Optional, Dict, List, Any
+from dataclasses import dataclass, field, asdict
+from enum import Enum
+from decimal import Decimal
+import croniter
+
+
+class TaskType(Enum):
+    """任务类型"""
+    ONCE = "once"        # 一次性任务
+    INTERVAL = "interval"  # 间隔任务
+    CRON = "cron"        # Cron表达式任务
+
+
+class TaskStatus(Enum):
+    """任务执行状态"""
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+    CANCELLED = "cancelled"
+
+
+@dataclass
+class ScheduledTask:
+    """定时任务模型"""
+    task_name: str                      # 要执行的函数名（对应@app.task注册的函数名）
+    task_type: TaskType                 # 任务类型
+    queue_name: str                     # 目标队列
+    
+    # 可选字段
+    id: Optional[int] = None            # 数据库自增ID（唯一标识）
+    scheduler_id: Optional[str] = None  # 任务的唯一标识符（用于去重）
+    namespace: str = 'default'          # 命名空间
+    task_args: List[Any] = field(default_factory=list)
+    task_kwargs: Dict[str, Any] = field(default_factory=dict)
+    cron_expression: Optional[str] = None
+    interval_seconds: Optional[float] = None
+    next_run_time: Optional[datetime] = None
+    last_run_time: Optional[datetime] = None
+    enabled: bool = True
+    max_retries: int = 3
+    retry_delay: int = 60
+    timeout: int = 300
+    priority: Optional[int] = None      # 任务优先级 (1=最高, 数字越大优先级越低，None=默认最低)
+    description: Optional[str] = None
+    tags: List[str] = field(default_factory=list)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+    
+    def __post_init__(self):
+        """初始化后处理"""
+        # 转换枚举类型
+        if isinstance(self.task_type, str):
+            self.task_type = TaskType(self.task_type)
+        
+        # 验证配置
+        self._validate()
+        
+        # 计算下次执行时间
+        if self.next_run_time is None:
+            self.next_run_time = self.calculate_next_run_time()
+    
+    def _validate(self):
+        """验证任务配置"""
+        if self.task_type == TaskType.CRON and not self.cron_expression:
+            raise ValueError(f"Task {self.task_name} with type CRON must have cron_expression")
+        
+        if self.task_type == TaskType.INTERVAL and not self.interval_seconds:
+            raise ValueError(f"Task {self.task_name} with type INTERVAL must have interval_seconds")
+        
+        # ONCE类型任务不应该有interval_seconds参数
+        if self.task_type == TaskType.ONCE and self.interval_seconds is not None:
+            raise ValueError(f"Task {self.task_name} with type ONCE should not have interval_seconds. Use next_run_time to specify when to run the task")
+    
+    def calculate_next_run_time(self, from_time: Optional[datetime] = None) -> Optional[datetime]:
+        """计算下次执行时间"""
+        if not self.enabled:
+            return None
+        
+        from_time = from_time or datetime.now()
+        
+        if self.task_type == TaskType.ONCE:
+            # 一次性任务，如果没有执行过就返回设定的时间
+            if self.last_run_time is None:
+                return self.next_run_time or from_time
+            return None
+        
+        elif self.task_type == TaskType.INTERVAL:
+            # 间隔任务
+            if self.last_run_time:
+                return self.last_run_time + timedelta(seconds=float(self.interval_seconds))
+            return from_time
+        
+        elif self.task_type == TaskType.CRON:
+            # Cron表达式任务
+            cron = croniter.croniter(self.cron_expression, from_time)
+            return cron.get_next(datetime)
+        
+        return None
+    
+    def update_next_run_time(self):
+        """更新下次执行时间"""
+        self.last_run_time = datetime.now()
+        self.next_run_time = self.calculate_next_run_time(from_time=self.last_run_time)
+    
+    def to_dict(self) -> dict:
+        """转换为字典"""
+        data = asdict(self)
+        data['task_type'] = self.task_type.value
+        
+        # 转换datetime为字符串
+        for key in ['next_run_time', 'last_run_time', 'created_at', 'updated_at']:
+            if data.get(key):
+                data[key] = data[key].isoformat() if isinstance(data[key], datetime) else data[key]
+        
+        # 转换Decimal为float
+        if data.get('interval_seconds') and isinstance(data['interval_seconds'], Decimal):
+            data['interval_seconds'] = float(data['interval_seconds'])
+        
+        return data
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> 'ScheduledTask':
+        """从字典创建实例"""
+        # 转换datetime字符串
+        for key in ['next_run_time', 'last_run_time', 'created_at', 'updated_at']:
+            if data.get(key) and isinstance(data[key], str):
+                data[key] = datetime.fromisoformat(data[key])
+        
+        # 转换task_type为枚举
+        if 'task_type' in data and isinstance(data['task_type'], str):
+            data['task_type'] = TaskType(data['task_type'])
+        
+        # 转换interval_seconds为float（处理Decimal类型）
+        if data.get('interval_seconds'):
+            if isinstance(data['interval_seconds'], Decimal):
+                data['interval_seconds'] = float(data['interval_seconds'])
+        
+        return cls(**data)
+    
+    def to_redis_value(self) -> str:
+        """转换为Redis存储的值"""
+        return json.dumps(self.to_dict())
+    
+    @classmethod
+    def from_redis_value(cls, value: str) -> 'ScheduledTask':
+        """从Redis值创建实例"""
+        return cls.from_dict(json.loads(value))
+
+
+@dataclass
+class TaskExecutionHistory:
+    """任务执行历史"""
+    task_id: int  # 对应 ScheduledTask 的 id
+    event_id: str
+    scheduled_time: datetime
+    status: TaskStatus
+    
+    # 可选字段
+    started_at: Optional[datetime] = None
+    finished_at: Optional[datetime] = None
+    result: Optional[Dict[str, Any]] = None
+    error_message: Optional[str] = None
+    retry_count: int = 0
+    duration_ms: Optional[int] = None
+    worker_id: Optional[str] = None
+    created_at: Optional[datetime] = None
+    
+    def __post_init__(self):
+        """初始化后处理"""
+        if isinstance(self.status, str):
+            self.status = TaskStatus(self.status)
+        
+        if self.created_at is None:
+            self.created_at = datetime.now()
+        
+        # 计算执行耗时
+        if self.started_at and self.finished_at and self.duration_ms is None:
+            delta = self.finished_at - self.started_at
+            self.duration_ms = int(delta.total_seconds() * 1000)
+    
+    def to_dict(self) -> dict:
+        """转换为字典"""
+        data = asdict(self)
+        data['status'] = self.status.value
+        
+        # 转换datetime为字符串
+        for key in ['scheduled_time', 'started_at', 'finished_at', 'created_at']:
+            if data.get(key):
+                data[key] = data[key].isoformat() if isinstance(data[key], datetime) else data[key]
+        
+        return data

jettask/scheduler/multi_namespace_scheduler.py

@@ -0,0 +1,294 @@
+"""
+多命名空间调度器管理器
+自动检测和管理多个命名空间的调度器实例
+"""
+import asyncio
+import logging
+import multiprocessing
+import traceback
+from typing import Dict, Set
+
+from jettask import Jettask
+from jettask.webui.task_center import TaskCenter
+from jettask.scheduler.scheduler import TaskScheduler
+from jettask.scheduler.manager import ScheduledTaskManager
+
+logger = logging.getLogger(__name__)
+
+
+class MultiNamespaceSchedulerManager:
+    """多命名空间调度器管理器"""
+    
+    def __init__(self, 
+                 task_center_base_url: str,
+                 check_interval: int = 30,
+                 scheduler_interval: float = 0.1,
+                 batch_size: int = 100,
+                 debug: bool = False):
+        """
+        初始化多命名空间调度器管理器
+        
+        Args:
+            task_center_base_url: 任务中心基础URL (如 http://localhost:8001)
+            check_interval: 命名空间检测间隔（秒）
+            scheduler_interval: 调度器扫描间隔（秒）
+            batch_size: 每批处理的最大任务数
+            debug: 是否启用调试模式
+        """
+        self.task_center_base_url = task_center_base_url.rstrip('/')
+        self.check_interval = check_interval
+        self.scheduler_interval = scheduler_interval
+        self.batch_size = batch_size
+        self.debug = debug
+        
+        # 存储每个命名空间的进程
+        self.scheduler_processes: Dict[str, multiprocessing.Process] = {}
+        self.running = False
+        
+        # 设置日志级别
+        if debug:
+            logging.basicConfig(level=logging.DEBUG)
+        else:
+            logging.basicConfig(level=logging.INFO)
+    
+    async def get_active_namespaces(self) -> Set[str]:
+        """获取所有活跃的命名空间"""
+        import aiohttp
+        
+        try:
+            # 直接调用API获取命名空间列表
+            url = f"{self.task_center_base_url}/api/namespaces"
+            
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as response:
+                    if response.status == 200:
+                        namespaces = await response.json()
+                        if namespaces:
+                            active_names = {ns['name'] for ns in namespaces if ns.get('name')}
+                            logger.info(f"发现 {len(active_names)} 个活跃的命名空间: {active_names}")
+                            return active_names
+                        else:
+                            logger.warning("没有找到任何命名空间")
+                            return set()
+                    else:
+                        logger.error(f"获取命名空间列表失败，状态码: {response.status}")
+                        return set()
+                        
+        except Exception as e:
+            logger.error(f"获取命名空间列表失败: {e}")
+            return set()
+    
+    def start_scheduler_for_namespace(self, namespace: str):
+        """为指定命名空间启动调度器进程"""
+        if namespace in self.scheduler_processes:
+            # 检查进程是否还在运行
+            if self.scheduler_processes[namespace].is_alive():
+                return
+            else:
+                # 清理已停止的进程
+                logger.info(f"清理已停止的调度器进程: {namespace}")
+                self.scheduler_processes[namespace].terminate()
+                self.scheduler_processes[namespace].join(timeout=5)
+                del self.scheduler_processes[namespace]
+        
+        # 创建新进程
+        process = multiprocessing.Process(
+            target=run_scheduler_for_namespace,
+            args=(
+                namespace,
+                self.task_center_base_url,
+                self.scheduler_interval,
+                self.batch_size,
+                self.debug
+            ),
+            name=f"scheduler_{namespace}"
+        )
+        
+        process.start()
+        self.scheduler_processes[namespace] = process
+        logger.info(f"启动命名空间 {namespace} 的调度器进程, PID: {process.pid}")
+    
+    def stop_scheduler_for_namespace(self, namespace: str):
+        """停止指定命名空间的调度器进程"""
+        if namespace in self.scheduler_processes:
+            process = self.scheduler_processes[namespace]
+            if process.is_alive():
+                logger.info(f"停止命名空间 {namespace} 的调度器进程")
+                process.terminate()
+                process.join(timeout=10)
+                
+                if process.is_alive():
+                    logger.warning(f"强制停止命名空间 {namespace} 的调度器进程")
+                    process.kill()
+                    process.join(timeout=5)
+            
+            del self.scheduler_processes[namespace]
+    
+    async def check_and_update_schedulers(self):
+        """检查并更新调度器（添加新的，停止已删除的）"""
+        # 获取当前活跃的命名空间
+        active_namespaces = await self.get_active_namespaces()
+        current_namespaces = set(self.scheduler_processes.keys())
+        
+        # 找出需要添加的命名空间
+        to_add = active_namespaces - current_namespaces
+        # 找出需要删除的命名空间
+        to_remove = current_namespaces - active_namespaces
+        
+        # 启动新的调度器
+        for namespace in to_add:
+            logger.info(f"检测到新命名空间: {namespace}")
+            self.start_scheduler_for_namespace(namespace)
+        
+        # 停止已删除的调度器
+        for namespace in to_remove:
+            logger.info(f"检测到命名空间已删除: {namespace}")
+            self.stop_scheduler_for_namespace(namespace)
+        
+        # 检查现有进程的健康状态
+        for namespace in active_namespaces & current_namespaces:
+            process = self.scheduler_processes.get(namespace)
+            if process and not process.is_alive():
+                logger.warning(f"调度器进程 {namespace} 已停止，重新启动")
+                del self.scheduler_processes[namespace]
+                self.start_scheduler_for_namespace(namespace)
+    
+    async def run(self):
+        """运行多命名空间调度器管理器"""
+        self.running = True
+        logger.info(f"启动多命名空间调度器管理器")
+        logger.info(f"任务中心: {self.task_center_base_url}")
+        logger.info(f"命名空间检测间隔: {self.check_interval} 秒")
+        logger.info(f"调度器扫描间隔: {self.scheduler_interval} 秒")
+        logger.info(f"批处理大小: {self.batch_size}")
+        
+        # 初始检查和启动
+        await self.check_and_update_schedulers()
+        
+        # 定期检查命名空间变化
+        while self.running:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self.check_and_update_schedulers()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"检查命名空间时出错: {e}")
+                if self.debug:
+                    traceback.print_exc()
+    
+    def stop(self):
+        """停止管理器和所有调度器"""
+        logger.info("停止多命名空间调度器管理器")
+        self.running = False
+        
+        # 停止所有调度器进程
+        for namespace in list(self.scheduler_processes.keys()):
+            self.stop_scheduler_for_namespace(namespace)
+        
+        logger.info("所有调度器已停止")
+
+
+def run_scheduler_for_namespace(namespace: str, 
+                               task_center_base_url: str,
+                               interval: float,
+                               batch_size: int,
+                               debug: bool):
+    """在独立进程中运行指定命名空间的调度器"""
+    import asyncio
+    import logging
+    import signal
+    import sys
+    
+    # 设置进程标题（如果可用）
+    try:
+        import setproctitle  # type: ignore
+        setproctitle.setproctitle(f"jettask-scheduler-{namespace}")
+    except ImportError:
+        pass
+    
+    # 配置日志
+    logging.basicConfig(
+        level=logging.DEBUG if debug else logging.INFO,
+        format=f'%(asctime)s - %(levelname)s - [{namespace}] %(message)s'
+    )
+    logger = logging.getLogger(__name__)
+    
+    async def run_scheduler():
+        """运行调度器的异步函数"""
+        scheduler_instance = None
+        try:
+            # 构建命名空间特定的URL
+            task_center_url = f"{task_center_base_url}/api/namespaces/{namespace}"
+            logger.info(f"连接到任务中心: {task_center_url}")
+            
+            # 连接任务中心
+            tc = TaskCenter(task_center_url)
+            if not tc._connect_sync():
+                logger.error(f"无法连接到任务中心: {namespace}")
+                return
+            
+            logger.info(f"成功连接到命名空间: {tc.namespace_name}")
+            
+            # 创建app实例
+            app = Jettask(task_center=tc)
+            
+            if not app.redis_url or not app.pg_url:
+                logger.error(f"任务中心配置不完整: {namespace}")
+                return
+            
+            # 显示配置信息
+            logger.info(f"命名空间 {namespace} 的调度器配置:")
+            logger.info(f"  Redis: {app.redis_url}")
+            logger.info(f"  PostgreSQL: {app.pg_url}")
+            logger.info(f"  间隔: {interval} 秒")
+            logger.info(f"  批大小: {batch_size}")
+            
+            # 创建调度器实例
+            manager = ScheduledTaskManager(app)
+            scheduler_instance = TaskScheduler(
+                app=app,
+                db_manager=manager,
+                scan_interval=interval,
+                batch_size=batch_size
+            )
+            
+            # 运行调度器（run方法内部会处理连接）
+            logger.info(f"启动命名空间 {namespace} 的调度器...")
+            await scheduler_instance.run()
+            
+        except asyncio.CancelledError:
+            logger.info(f"调度器 {namespace} 收到取消信号")
+        except KeyboardInterrupt:
+            logger.info(f"调度器 {namespace} 收到中断信号")
+        except Exception as e:
+            logger.error(f"调度器 {namespace} 运行错误: {e}")
+            if debug:
+                traceback.print_exc()
+        finally:
+            # 清理资源
+            if scheduler_instance:
+                scheduler_instance.stop()
+            
+            logger.info(f"调度器 {namespace} 已停止")
+    
+    # 设置信号处理
+    import signal
+    import sys
+    
+    def signal_handler(signum, frame):
+        logger.info(f"调度器 {namespace} 收到信号 {signum}")
+        sys.exit(0)
+    
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+    
+    # 运行调度器
+    try:
+        asyncio.run(run_scheduler())
+    except (KeyboardInterrupt, SystemExit):
+        logger.info(f"调度器 {namespace} 正常退出")
+    except Exception as e:
+        logger.error(f"调度器 {namespace} 异常退出: {e}")
+        if debug:
+            traceback.print_exc()

jettask/scheduler/performance_optimization.sql

@@ -0,0 +1,45 @@
+-- Performance optimization for scheduled_tasks table
+-- This migration adds necessary indexes for optimal query performance
+
+-- 1. Unique index on scheduler_id (已经存在，但确保创建)
+CREATE UNIQUE INDEX IF NOT EXISTS idx_scheduled_tasks_scheduler_id 
+ON scheduled_tasks(scheduler_id);
+
+-- 2. Index for ready tasks query (get_ready_tasks)
+-- 这是最频繁的查询之一，需要复合索引
+CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_ready 
+ON scheduled_tasks(next_run_time, enabled) 
+WHERE enabled = true AND next_run_time IS NOT NULL;
+
+-- 3. Index for task listing with filters (list_tasks)
+-- created_at用于排序
+CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_created 
+ON scheduled_tasks(created_at DESC);
+
+-- 4. Index for task_name lookups (常用于查找特定任务)
+CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_name 
+ON scheduled_tasks(task_name);
+
+-- 5. Composite index for enabled tasks with type
+CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_enabled_type 
+ON scheduled_tasks(enabled, task_type) 
+WHERE enabled = true;
+
+-- 6. Index for update operations by id (primary key已自动有索引)
+-- 但确保主键约束存在
+-- (主键自动创建索引，无需额外操作)
+
+-- 7. Index for task execution history queries
+CREATE INDEX IF NOT EXISTS idx_task_history_task_scheduled 
+ON task_execution_history(task_id, scheduled_time DESC);
+
+-- 8. Index for history status queries
+CREATE INDEX IF NOT EXISTS idx_task_history_status_created 
+ON task_execution_history(status, created_at DESC);
+
+-- 添加表注释
+COMMENT ON TABLE scheduled_tasks IS 'Scheduled tasks configuration table with optimized indexes';
+
+-- 分析表以更新统计信息
+ANALYZE scheduled_tasks;
+ANALYZE task_execution_history;