jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/scheduler/make_scheduler_id_required.sql

@@ -1,28 +0,0 @@
--- Migration to make scheduler_id NOT NULL
--- First, update any NULL scheduler_id values with generated ones
-
--- Update NULL scheduler_id values with generated unique IDs
-UPDATE scheduled_tasks 
-SET scheduler_id = CONCAT(task_name, '_', task_type, '_', id)
-WHERE scheduler_id IS NULL;
-
--- Now make the column NOT NULL
-ALTER TABLE scheduled_tasks 
-ALTER COLUMN scheduler_id SET NOT NULL;
-
--- Ensure the UNIQUE constraint exists
-DO $$ 
-BEGIN
-    IF NOT EXISTS (
-        SELECT 1 
-        FROM pg_constraint 
-        WHERE conname = 'scheduled_tasks_scheduler_id_key'
-    ) THEN
-        ALTER TABLE scheduled_tasks 
-        ADD CONSTRAINT scheduled_tasks_scheduler_id_key UNIQUE (scheduler_id);
-    END IF;
-END $$;
-
--- Update the comment
-COMMENT ON COLUMN scheduled_tasks.scheduler_id IS 
-'Unique identifier for the task (required, used for deduplication)';

jettask/scheduler/migrate_interval_seconds.sql

@@ -1,9 +0,0 @@
--- Migration to change interval_seconds from INTEGER to NUMERIC
--- This allows storing decimal values like 0.1 seconds
-
--- Alter the column type
-ALTER TABLE scheduled_tasks 
-ALTER COLUMN interval_seconds TYPE NUMERIC(10,2);
-
--- Add a comment to document the change
-COMMENT ON COLUMN scheduled_tasks.interval_seconds IS 'Interval in seconds for interval-type tasks (supports decimal values)';

jettask/scheduler/performance_optimization.sql

@@ -1,45 +0,0 @@
--- Performance optimization for scheduled_tasks table
--- This migration adds necessary indexes for optimal query performance
-
--- 1. Unique index on scheduler_id (已经存在，但确保创建)
-CREATE UNIQUE INDEX IF NOT EXISTS idx_scheduled_tasks_scheduler_id 
-ON scheduled_tasks(scheduler_id);
-
--- 2. Index for ready tasks query (get_ready_tasks)
--- 这是最频繁的查询之一，需要复合索引
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_ready 
-ON scheduled_tasks(next_run_time, enabled) 
-WHERE enabled = true AND next_run_time IS NOT NULL;
-
--- 3. Index for task listing with filters (list_tasks)
--- created_at用于排序
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_created 
-ON scheduled_tasks(created_at DESC);
-
--- 4. Index for task_name lookups (常用于查找特定任务)
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_name 
-ON scheduled_tasks(task_name);
-
--- 5. Composite index for enabled tasks with type
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_enabled_type 
-ON scheduled_tasks(enabled, task_type) 
-WHERE enabled = true;
-
--- 6. Index for update operations by id (primary key已自动有索引)
--- 但确保主键约束存在
--- (主键自动创建索引，无需额外操作)
-
--- 7. Index for task execution history queries
-CREATE INDEX IF NOT EXISTS idx_task_history_task_scheduled 
-ON task_execution_history(task_id, scheduled_time DESC);
-
--- 8. Index for history status queries
-CREATE INDEX IF NOT EXISTS idx_task_history_status_created 
-ON task_execution_history(status, created_at DESC);
-
--- 添加表注释
-COMMENT ON TABLE scheduled_tasks IS 'Scheduled tasks configuration table with optimized indexes';
-
--- 分析表以更新统计信息
-ANALYZE scheduled_tasks;
-ANALYZE task_execution_history;

jettask/scheduler/run_scheduler.py

@@ -1,186 +0,0 @@
-#!/usr/bin/env python
-"""
-定时任务调度器启动脚本
-"""
-import asyncio
-import os
-import sys
-import signal
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent.parent
-sys.path.insert(0, str(project_root))
-
-from jettask import Jettask
-from jettask.scheduler.scheduler import TaskScheduler
-from jettask.scheduler.task_crud import ScheduledTaskManager
-from jettask.utils.task_logger import get_task_logger
-
-logger = get_task_logger(__name__)
-
-
-class SchedulerRunner:
-    """调度器运行器"""
-    
-    def __init__(self):
-        self.scheduler = None
-        self.app = None
-        self.running = False
-    
-    async def setup(self):
-        """初始化设置"""
-        # 从环境变量获取配置
-        redis_url = os.getenv('JETTASK_REDIS_URL', 'redis://localhost:6379/0')
-        db_url = os.getenv('JETTASK_PG_URL', 'postgresql://jettask:123456@localhost:5432/jettask')
-        
-        # 创建Jettask应用实例
-        self.app = Jettask(
-            redis_url=redis_url,
-            pg_url=db_url
-        )
-        
-        # 导入任务模块（确保任务被注册）
-        try:
-            # 尝试导入用户定义的任务模块
-            import importlib
-            import importlib.util
-            
-            # 先尝试加载示例模块
-            example_paths = [
-                'examples.scheduler_auto_demo',
-                'scheduler_auto_demo'
-            ]
-            
-            for path in example_paths:
-                try:
-                    example_module = importlib.import_module(path)
-                    if hasattr(example_module, 'app'):
-                        # 将任务模块中的任务复制到调度器的app中
-                        for task_name, task_obj in example_module.app._tasks.items():
-                            self.app._tasks[task_name] = task_obj
-                            # 也注册带模块名的任务
-                            full_name = f"scheduler_auto_demo.{task_name}"
-                            self.app._tasks[full_name] = task_obj
-                            logger.info(f"Registered task: {task_name} and {full_name}")
-                        logger.info(f"Loaded example tasks from module: {path}")
-                        break
-                except ImportError:
-                    continue
-            
-            # 然后加载用户指定的任务模块
-            task_module = os.getenv('JETTASK_TASKS_MODULE', 'tasks')
-            if task_module:
-                try:
-                    module = importlib.import_module(task_module)
-                    # 获取模块中的app对象的任务
-                    if hasattr(module, 'app'):
-                        # 将任务模块中的任务复制到调度器的app中
-                        for task_name, task_obj in module.app._tasks.items():
-                            self.app._tasks[task_name] = task_obj
-                            logger.info(f"Registered task: {task_name}")
-                    
-                    # 注册别名任务（scheduler_auto_demo.send_notification）
-                    if hasattr(module, 'send_notification'):
-                        self.app._tasks['scheduler_auto_demo.send_notification'] = module.send_notification
-                        logger.info("Registered task: scheduler_auto_demo.send_notification")
-                    
-                    logger.info(f"Loaded tasks from module: {task_module}")
-                except ImportError as e:
-                    logger.warning(f"Could not import tasks module '{task_module}': {e}")
-        except Exception as e:
-            logger.warning(f"Error loading tasks: {e}")
-        
-        # 创建数据库管理器
-        db_manager = ScheduledTaskManager(db_url)
-        await db_manager.connect()
-        
-        # 确保数据库表存在
-        await db_manager.init_schema()
-        
-        # 创建调度器
-        self.scheduler = TaskScheduler(
-            app=self.app,
-            redis_url=redis_url,
-            db_manager=db_manager,
-            redis_prefix="jettask:scheduled",
-            scan_interval=0.5,  # 每0.5秒扫描一次
-            batch_size=100,      # 每批处理100个任务
-            leader_ttl=30        # Leader锁30秒过期
-        )
-        
-        await self.scheduler.connect()
-        logger.info("Scheduler setup completed")
-    
-    async def run(self):
-        """运行调度器"""
-        self.running = True
-        logger.info("Starting scheduler...")
-        
-        try:
-            # 运行调度器
-            await self.scheduler.run()
-        except Exception as e:
-            logger.error(f"Scheduler error: {e}", exc_info=True)
-        finally:
-            self.running = False
-            logger.info("Scheduler stopped")
-    
-    async def shutdown(self):
-        """关闭调度器"""
-        logger.info("Shutting down scheduler...")
-        
-        if self.scheduler:
-            self.scheduler.stop()
-            await self.scheduler.disconnect()
-        
-        if self.app:
-            # 关闭应用连接
-            pass
-        
-        logger.info("Scheduler shutdown completed")
-    
-    def handle_signal(self, signum, frame):
-        """处理终止信号"""
-        logger.info(f"Received signal {signum}, initiating shutdown...")
-        self.running = False
-        
-        # 创建关闭任务
-        if self.scheduler:
-            self.scheduler.stop()
-
-
-async def main():
-    """主函数"""
-    runner = SchedulerRunner()
-    
-    # 设置信号处理
-    signal.signal(signal.SIGINT, runner.handle_signal)
-    signal.signal(signal.SIGTERM, runner.handle_signal)
-    
-    try:
-        # 初始化
-        await runner.setup()
-        
-        # 运行调度器
-        await runner.run()
-        
-    except KeyboardInterrupt:
-        logger.info("Received keyboard interrupt")
-    except Exception as e:
-        logger.error(f"Unexpected error: {e}", exc_info=True)
-    finally:
-        # 清理
-        await runner.shutdown()
-
-
-if __name__ == '__main__':
-    # 设置日志级别
-    import logging
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-    )
-    
-    # 运行主函数
-    asyncio.run(main())

jettask/scheduler/schema.sql

@@ -1,84 +0,0 @@
--- 定时任务表
-CREATE TABLE IF NOT EXISTS scheduled_tasks (
-    id BIGSERIAL PRIMARY KEY,               -- 自增主键（任务唯一标识）
-    scheduler_id VARCHAR(255) NOT NULL UNIQUE, -- 任务的唯一标识符（必填，用于去重）
-    task_name VARCHAR(255) NOT NULL,        -- 要执行的函数名（对应@app.task注册的任务名）
-    task_type VARCHAR(50) NOT NULL,         -- 任务类型: cron, interval, once
-    
-    -- 任务执行相关
-    queue_name VARCHAR(100) NOT NULL,       -- 目标队列名
-    task_args JSONB DEFAULT '[]',           -- 任务参数
-    task_kwargs JSONB DEFAULT '{}',         -- 任务关键字参数
-    
-    -- 调度相关
-    cron_expression VARCHAR(100),           -- cron表达式 (task_type=cron时使用)
-    interval_seconds NUMERIC(10,2),         -- 间隔秒数 (task_type=interval时使用，支持小数)
-    next_run_time TIMESTAMP WITH TIME ZONE, -- 下次执行时间
-    last_run_time TIMESTAMP WITH TIME ZONE, -- 上次执行时间
-    
-    -- 状态和控制
-    enabled BOOLEAN DEFAULT true,           -- 是否启用
-    max_retries INTEGER DEFAULT 3,          -- 最大重试次数
-    retry_delay INTEGER DEFAULT 60,         -- 重试延迟(秒)
-    timeout INTEGER DEFAULT 300,            -- 任务超时时间(秒)
-    priority INTEGER DEFAULT NULL,          -- 任务优先级 (1=最高, 数字越大优先级越低，NULL=默认最低)
-    
-    -- 元数据
-    description TEXT,                       -- 任务描述
-    tags JSONB DEFAULT '[]',                -- 标签
-    metadata JSONB DEFAULT '{}',            -- 额外元数据
-    
-    -- 时间戳
-    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
-    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
-);
-
--- 索引
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_time) WHERE enabled = true;
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_task_type ON scheduled_tasks(task_type);
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_queue ON scheduled_tasks(queue_name);
-CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_enabled ON scheduled_tasks(enabled);
-CREATE UNIQUE INDEX IF NOT EXISTS idx_scheduled_tasks_scheduler_id ON scheduled_tasks(scheduler_id);
-
--- 任务执行历史表
-CREATE TABLE IF NOT EXISTS task_execution_history (
-    id BIGSERIAL PRIMARY KEY,
-    task_id BIGINT NOT NULL,                -- 关联的任务ID（外键到 scheduled_tasks.id）
-    event_id VARCHAR(255) NOT NULL,         -- 执行事件ID
-    
-    -- 执行信息
-    scheduled_time TIMESTAMP WITH TIME ZONE NOT NULL,  -- 计划执行时间
-    started_at TIMESTAMP WITH TIME ZONE,               -- 实际开始时间
-    finished_at TIMESTAMP WITH TIME ZONE,              -- 完成时间
-    
-    -- 执行结果
-    status VARCHAR(50) NOT NULL,            -- pending, running, success, failed, timeout
-    result JSONB,                           -- 执行结果
-    error_message TEXT,                     -- 错误信息
-    retry_count INTEGER DEFAULT 0,          -- 重试次数
-    
-    -- 性能指标
-    duration_ms INTEGER,                    -- 执行耗时(毫秒)
-    worker_id VARCHAR(100),                 -- 执行的worker ID
-    
-    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
-);
-
--- 索引
-CREATE INDEX IF NOT EXISTS idx_task_history_task_id ON task_execution_history(task_id);
-CREATE INDEX IF NOT EXISTS idx_task_history_event_id ON task_execution_history(event_id);
-CREATE INDEX IF NOT EXISTS idx_task_history_status ON task_execution_history(status);
-CREATE INDEX IF NOT EXISTS idx_task_history_scheduled ON task_execution_history(scheduled_time);
-CREATE INDEX IF NOT EXISTS idx_task_history_created ON task_execution_history(created_at);
-
--- 更新时间触发器
-CREATE OR REPLACE FUNCTION update_updated_at_column()
-RETURNS TRIGGER AS $$
-BEGIN
-    NEW.updated_at = CURRENT_TIMESTAMP;
-    RETURN NEW;
-END;
-$$ language 'plpgsql';
-
-CREATE TRIGGER update_scheduled_tasks_updated_at BEFORE UPDATE
-    ON scheduled_tasks FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

jettask/task/task_executor.py

@@ -1,318 +0,0 @@
-"""
-任务执行器
-
-负责任务的执行、超时控制、错误处理
-"""
-
-from typing import List, Dict, Any, Optional
-import asyncio
-import logging
-import traceback
-from datetime import datetime
-
-from .task_registry import TaskRegistry, TaskDefinition
-
-logger = logging.getLogger('app')
-
-
-class TaskExecutor:
-    """
-    任务执行器
-
-    职责：
-    1. 任务执行
-    2. 超时控制
-    3. 错误处理
-    4. 结果存储
-
-    从ExecutorCore提取的纯任务执行逻辑
-    """
-
-    def __init__(self,
-                 task_registry: TaskRegistry,
-                 data_access=None,
-                 retry_manager=None):
-        """
-        初始化任务执行器
-
-        Args:
-            task_registry: 任务注册器
-            data_access: 数据访问层（可选）
-            retry_manager: 重试管理器（可选）
-        """
-        self.registry = task_registry
-        self.data_access = data_access
-        self.retry_manager = retry_manager
-
-        logger.debug("TaskExecutor initialized")
-
-    async def execute(self, message: dict) -> Any:
-        """
-        执行任务
-
-        Args:
-            message: 任务消息
-                - task_name: 任务名称
-                - task_id: 任务ID
-                - args: 位置参数
-                - kwargs: 关键字参数
-
-        Returns:
-            Any: 任务执行结果
-
-        Raises:
-            ValueError: 任务不存在
-            asyncio.TimeoutError: 任务超时
-            Exception: 任务执行错误
-        """
-        task_name = message.get('task_name')
-        task_id = message.get('task_id')
-        args = message.get('args', [])
-        kwargs = message.get('kwargs', {})
-
-        logger.debug(f"Executing task: {task_name} (id: {task_id})")
-
-        # 获取任务定义
-        task_def = self.registry.get(task_name)
-        if not task_def:
-            error_msg = f"Task {task_name} not found in registry"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 执行任务（带超时）
-        try:
-            # 记录开始时间
-            start_time = datetime.now()
-
-            # 执行任务
-            if asyncio.iscoroutinefunction(task_def.func):
-                # 异步任务
-                result = await asyncio.wait_for(
-                    task_def.func(*args, **kwargs),
-                    timeout=task_def.timeout
-                )
-            else:
-                # 同步任务，在线程池中执行
-                loop = asyncio.get_event_loop()
-                result = await asyncio.wait_for(
-                    loop.run_in_executor(None, task_def.func, *args, **kwargs),
-                    timeout=task_def.timeout
-                )
-
-            # 计算执行时间
-            execution_time = (datetime.now() - start_time).total_seconds()
-
-            logger.info(
-                f"Task {task_name} (id: {task_id}) completed "
-                f"in {execution_time:.2f}s"
-            )
-
-            # 保存成功结果
-            if self.data_access:
-                await self.data_access.save_task_result(
-                    task_id=task_id,
-                    status='completed',
-                    result=result,
-                    execution_time=execution_time
-                )
-
-            return result
-
-        except asyncio.TimeoutError:
-            # 超时处理
-            logger.error(
-                f"Task {task_name} (id: {task_id}) timeout "
-                f"after {task_def.timeout}s"
-            )
-            await self._handle_timeout(task_id, task_def, message)
-            raise
-
-        except Exception as e:
-            # 错误处理
-            logger.error(
-                f"Task {task_name} (id: {task_id}) failed: {e}\n"
-                f"{traceback.format_exc()}"
-            )
-            await self._handle_error(task_id, task_def, message, e)
-            raise
-
-    async def _handle_timeout(self,
-                             task_id: str,
-                             task_def: TaskDefinition,
-                             message: dict):
-        """
-        处理超时
-
-        Args:
-            task_id: 任务ID
-            task_def: 任务定义
-            message: 任务消息
-        """
-        # 保存超时状态
-        if self.data_access:
-            await self.data_access.save_task_result(
-                task_id=task_id,
-                status='timeout',
-                result=None,
-                error=f"Task timeout after {task_def.timeout}s"
-            )
-
-        # 重试逻辑
-        if task_def.max_retries > 0 and self.retry_manager:
-            current_retry = message.get('retry_count', 0)
-            if current_retry < task_def.max_retries:
-                logger.info(
-                    f"Scheduling retry for task {task_def.name} "
-                    f"(attempt {current_retry + 1}/{task_def.max_retries})"
-                )
-                await self.retry_manager.schedule_retry(
-                    task_id=task_id,
-                    task_def=task_def,
-                    message=message,
-                    retry_count=current_retry + 1,
-                    delay=task_def.retry_delay
-                )
-
-    async def _handle_error(self,
-                           task_id: str,
-                           task_def: TaskDefinition,
-                           message: dict,
-                           error: Exception):
-        """
-        处理错误
-
-        Args:
-            task_id: 任务ID
-            task_def: 任务定义
-            message: 任务消息
-            error: 错误对象
-        """
-        # 保存失败状态
-        if self.data_access:
-            await self.data_access.save_task_result(
-                task_id=task_id,
-                status='failed',
-                result=None,
-                error=str(error),
-                traceback=traceback.format_exc()
-            )
-
-        # 重试逻辑
-        if task_def.max_retries > 0 and self.retry_manager:
-            current_retry = message.get('retry_count', 0)
-            if current_retry < task_def.max_retries:
-                logger.info(
-                    f"Scheduling retry for task {task_def.name} "
-                    f"(attempt {current_retry + 1}/{task_def.max_retries})"
-                )
-                await self.retry_manager.schedule_retry(
-                    task_id=task_id,
-                    task_def=task_def,
-                    message=message,
-                    retry_count=current_retry + 1,
-                    delay=task_def.retry_delay
-                )
-            else:
-                logger.error(
-                    f"Task {task_def.name} failed after {task_def.max_retries} retries"
-                )
-
-    async def batch_update_status(self, status_updates: List[dict]):
-        """
-        批量更新任务状态
-
-        Args:
-            status_updates: 状态更新列表
-                - task_id: 任务ID
-                - status: 状态
-                - result: 结果（可选）
-                - error: 错误信息（可选）
-        """
-        if not self.data_access:
-            logger.warning("No data_access configured, skipping batch status update")
-            return
-
-        for update in status_updates:
-            try:
-                await self.data_access.update_task_status(
-                    task_id=update['task_id'],
-                    status=update['status'],
-                    result=update.get('result'),
-                    error=update.get('error')
-                )
-            except Exception as e:
-                logger.error(f"Error updating task status: {e}")
-
-    async def execute_batch(self, messages: List[dict]) -> List[Dict[str, Any]]:
-        """
-        批量执行任务
-
-        Args:
-            messages: 任务消息列表
-
-        Returns:
-            List[Dict]: 执行结果列表
-                - task_id: 任务ID
-                - status: 状态（completed/failed/timeout）
-                - result: 结果（可选）
-                - error: 错误信息（可选）
-        """
-        results = []
-
-        # 并发执行所有任务
-        tasks = []
-        for message in messages:
-            task = asyncio.create_task(self._execute_with_result(message))
-            tasks.append(task)
-
-        # 等待所有任务完成
-        task_results = await asyncio.gather(*tasks, return_exceptions=True)
-
-        # 整理结果
-        for i, task_result in enumerate(task_results):
-            message = messages[i]
-            task_id = message.get('task_id')
-
-            if isinstance(task_result, Exception):
-                # 任务失败
-                if isinstance(task_result, asyncio.TimeoutError):
-                    status = 'timeout'
-                    error = 'Task timeout'
-                else:
-                    status = 'failed'
-                    error = str(task_result)
-
-                results.append({
-                    'task_id': task_id,
-                    'status': status,
-                    'result': None,
-                    'error': error
-                })
-            else:
-                # 任务成功
-                results.append({
-                    'task_id': task_id,
-                    'status': 'completed',
-                    'result': task_result,
-                    'error': None
-                })
-
-        return results
-
-    async def _execute_with_result(self, message: dict) -> Any:
-        """
-        执行任务并返回结果（用于批量执行）
-
-        Args:
-            message: 任务消息
-
-        Returns:
-            Any: 任务执行结果
-
-        Raises:
-            Exception: 任务执行错误
-        """
-        try:
-            return await self.execute(message)
-        except Exception:
-            raise  # 让gather捕获异常