jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/scheduler/__init__.py

@@ -1,17 +1,29 @@
 """
 定时任务调度模块
-支持 Redis + PostgreSQL 双存储方案
+
+文件说明：
+- models.py: 数据模型（ScheduledTask, TaskExecutionHistory）
+- schedule.py: Schedule 定义类（用于定义定时任务）
+- scheduler.py: 核心调度器（TaskScheduler）
+- loader.py: 任务加载器（TaskLoader）
+- database.py: 数据库操作（ScheduledTaskManager）
+- manager.py: 统一调度器管理器（UnifiedSchedulerManager，用于CLI）
+- sql/: SQL文件目录
+  - schema.sql: 数据库表结构
+  - migrations/: 数据库迁移脚本
 """
 
 from .models import ScheduledTask, TaskExecutionHistory
 from .scheduler import TaskScheduler
 from .loader import TaskLoader
-from .task_crud import ScheduledTaskManager
+from .database import ScheduledTaskManager
+from .schedule import Schedule
 
 __all__ = [
     'ScheduledTask',
     'TaskExecutionHistory',
     'TaskScheduler',
     'TaskLoader',
-    'ScheduledTaskManager'
+    'ScheduledTaskManager',
+    'Schedule'
 ]

jettask/scheduler/{task_crud.py → database.py}

@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
 import json
 
 from .models import ScheduledTask, TaskExecutionHistory, TaskType, TaskStatus
+from jettask.db.connector import get_asyncpg_pool
 
 
 class ScheduledTaskManager:
@@ -34,14 +35,24 @@ class ScheduledTaskManager:
         
         self.pool: Optional[asyncpg.Pool] = None
     
-    async def connect(self):
-        """建立数据库连接池"""
+    async def connect(self, max_retries: int = 3, retry_delay: int = 5):
+        """
+        建立数据库连接池（使用统一的连接池管理）
+
+        Args:
+            max_retries: 最大重试次数，默认3次
+            retry_delay: 重试间隔（秒），默认5秒
+        """
         if not self.pool:
-            self.pool = await asyncpg.create_pool(
-                self.db_url,
+            # 使用统一的 get_asyncpg_pool 函数，它内部已经包含了重试机制和日志
+            self.pool = await get_asyncpg_pool(
+                dsn=self.db_url,
                 min_size=2,
                 max_size=10,
-                command_timeout=60
+                command_timeout=60,
+                timeout=10,  # 连接超时10秒
+                max_retries=max_retries,
+                retry_delay=retry_delay
             )
     
     async def disconnect(self):
@@ -53,7 +64,7 @@ class ScheduledTaskManager:
     async def init_schema(self):
         """初始化数据库表结构（幂等操作）"""
         import os
-        schema_path = os.path.join(os.path.dirname(__file__), 'schema.sql')
+        schema_path = os.path.join(os.path.dirname(__file__), 'sql', 'schema.sql')
         
         with open(schema_path, 'r') as f:
             schema_sql = f.read()
@@ -76,19 +87,18 @@ class ScheduledTaskManager:
         """创建定时任务"""
         sql = """
             INSERT INTO scheduled_tasks (
-                scheduler_id, task_name, task_type, queue_name, namespace,
+                scheduler_id, task_type, queue_name, namespace,
                 task_args, task_kwargs, cron_expression, interval_seconds,
                 next_run_time, enabled, max_retries, retry_delay, timeout,
                 priority, description, tags, metadata
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
             RETURNING *
         """
-        
+
         async with self.pool.acquire() as conn:
             row = await conn.fetchrow(
                 sql,
                 task.scheduler_id,
-                task.task_name,
                 task.task_type.value,
                 task.queue_name,
                 task.namespace,  # 添加namespace
@@ -106,7 +116,7 @@ class ScheduledTaskManager:
                 json.dumps(task.tags),
                 json.dumps(task.metadata)
             )
-            
+
             return self._row_to_task(row)
     
     async def get_task(self, task_id: int) -> Optional[ScheduledTask]:
@@ -134,34 +144,32 @@ class ScheduledTaskManager:
         sql = """
             UPDATE scheduled_tasks SET
                 scheduler_id = $2,
-                task_name = $3,
-                task_type = $4,
-                queue_name = $5,
-                namespace = $6,
-                task_args = $7,
-                task_kwargs = $8,
-                cron_expression = $9,
-                interval_seconds = $10,
-                next_run_time = $11,
-                last_run_time = $12,
-                enabled = $13,
-                max_retries = $14,
-                retry_delay = $15,
-                timeout = $16,
-                priority = $17,
-                description = $18,
-                tags = $19,
-                metadata = $20
+                task_type = $3,
+                queue_name = $4,
+                namespace = $5,
+                task_args = $6,
+                task_kwargs = $7,
+                cron_expression = $8,
+                interval_seconds = $9,
+                next_run_time = $10,
+                last_run_time = $11,
+                enabled = $12,
+                max_retries = $13,
+                retry_delay = $14,
+                timeout = $15,
+                priority = $16,
+                description = $17,
+                tags = $18,
+                metadata = $19
             WHERE id = $1
             RETURNING *
         """
-        
+
         async with self.pool.acquire() as conn:
             row = await conn.fetchrow(
                 sql,
                 task.id,
                 task.scheduler_id,
-                task.task_name,
                 task.task_type.value,
                 task.queue_name,
                 task.namespace,  # 添加namespace
@@ -300,10 +308,9 @@ class ScheduledTaskManager:
             return
         
         sql = """
-            UPDATE scheduled_tasks 
-            SET next_run_time = u.next_run_time, 
-                last_run_time = u.last_run_time,
-                execution_count = COALESCE(execution_count, 0) + 1
+            UPDATE scheduled_tasks
+            SET next_run_time = u.next_run_time,
+                last_run_time = u.last_run_time
             FROM (VALUES ($1::int, $2::timestamptz, $3::timestamptz)) AS u(id, next_run_time, last_run_time)
             WHERE scheduled_tasks.id = u.id
         """
@@ -439,12 +446,12 @@ class ScheduledTaskManager:
         for history in histories:
             task_info = task_info_map.get(history.task_id, {})
             queue_name = task_info.get('queue_name', 'default')
-            task_name = task_info.get('task_name', 'scheduled_task')
-            
+            # task_name 字段已移除，改为 queue_name
+
             data.append((
                 history.event_id,  # id字段使用event_id
                 queue_name,
-                task_name,  # task_name
+                queue_name,  # task_name（保留以兼容旧表结构，后续可删除）
                 history.status.value if isinstance(history.status, TaskStatus) else history.status,
                 history.task_id,  # scheduled_task_id
                 history.scheduled_time,
@@ -469,25 +476,24 @@ class ScheduledTaskManager:
         sql = """
             UPDATE scheduled_tasks SET
                 scheduler_id = $2,
-                task_name = $3,
-                task_type = $4,
-                queue_name = $5,
-                task_args = $6,
-                task_kwargs = $7,
-                cron_expression = $8,
-                interval_seconds = $9,
-                next_run_time = $10,
-                last_run_time = $11,
-                enabled = $12,
-                max_retries = $13,
-                retry_delay = $14,
-                timeout = $15,
-                description = $16,
-                metadata = $17,
-                updated_at = $18
+                task_type = $3,
+                queue_name = $4,
+                task_args = $5,
+                task_kwargs = $6,
+                cron_expression = $7,
+                interval_seconds = $8,
+                next_run_time = $9,
+                last_run_time = $10,
+                enabled = $11,
+                max_retries = $12,
+                retry_delay = $13,
+                timeout = $14,
+                description = $15,
+                metadata = $16,
+                updated_at = $17
             WHERE id = $1
         """
-        
+
         # 准备批量数据
         data = []
         now = datetime.now()
@@ -495,7 +501,6 @@ class ScheduledTaskManager:
             data.append((
                 task.id,
                 task.scheduler_id,
-                task.task_name,
                 task.task_type.value if isinstance(task.task_type, TaskType) else task.task_type,
                 task.queue_name,
                 json.dumps(task.task_args) if task.task_args else '[]',
@@ -531,7 +536,6 @@ class ScheduledTaskManager:
         return ScheduledTask(
             id=row['id'],
             scheduler_id=row['scheduler_id'],
-            task_name=row['task_name'],
             task_type=TaskType(row['task_type']),
             queue_name=row['queue_name'],
             namespace=row.get('namespace', 'default'),  # 添加namespace字段

jettask/scheduler/loader.py

@@ -8,7 +8,7 @@ from datetime import datetime, timedelta
 import json
 
 from ..utils.task_logger import get_task_logger, LogContext
-from .task_crud import ScheduledTaskManager
+from .database import ScheduledTaskManager
 from .models import ScheduledTask
 
 
@@ -55,7 +55,7 @@ class TaskLoader:
     async def connect(self):
         """建立Redis连接（使用统一的连接池管理）"""
         if not self.redis:
-            from jettask.utils.db_connector import get_async_redis_client
+            from jettask.db.connector import get_async_redis_client
 
             self.redis = get_async_redis_client(
                 redis_url=self.redis_url,

jettask/scheduler/{scheduler_coordinator.py → manager.py}

@@ -10,7 +10,7 @@ from jettask.core.unified_manager_base import UnifiedManagerBase
 from jettask import Jettask
 from jettask.task.task_center.client import TaskCenter
 from .scheduler import TaskScheduler
-from .task_crud import ScheduledTaskManager
+from .database import ScheduledTaskManager
 
 logger = logging.getLogger(__name__)
 
@@ -50,20 +50,32 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
     async def run_single_namespace(self, namespace_name: str):
         """
         运行单命名空间模式
-        
+
         Args:
             namespace_name: 命名空间名称
         """
         logger.info(f"启动单命名空间调度器: {namespace_name}")
         logger.info(f"扫描间隔: {self.scan_interval}秒")
         logger.info(f"批处理大小: {self.batch_size}")
-        
+
         try:
+            # 构建任务中心URL
+            # 如果URL已经是完整的API格式，直接使用；否则需要转换为标准格式
+            if '/api/v1/namespaces/' in self.task_center_url or '/api/namespaces/' in self.task_center_url:
+                # 已经是完整的API格式
+                namespace_url = self.task_center_url
+            else:
+                # 简化格式，需要转换为标准API格式
+                base_url = self.get_base_url()
+                namespace_url = f"{base_url}/api/v1/namespaces/{namespace_name}"
+
+            logger.debug(f"任务中心API URL: {namespace_url}")
+
             # 创建任务中心连接
-            tc = TaskCenter(self.task_center_url)
+            tc = TaskCenter(namespace_url)
             if not tc._connect_sync():
-                raise Exception(f"无法连接到任务中心: {self.task_center_url}")
-            
+                raise Exception(f"无法连接到任务中心: {namespace_url}")
+
             # 创建 Jettask 应用
             app = Jettask(task_center=tc)
 
@@ -175,6 +187,7 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
                 # 1. http://localhost:8001 -> http://localhost:8001/api/v1/namespaces/{name}
                 # 2. http://localhost:8001/api/v1 -> http://localhost:8001/api/v1/namespaces/{name}
                 # 3. http://localhost:8001/api/v1/namespaces/old -> http://localhost:8001/api/v1/namespaces/{name}
+                # 4. http://localhost:8001/namespaces/old -> http://localhost:8001/api/v1/namespaces/{name}
 
                 if '/api/v1/namespaces/' in task_center_url:
                     # 替换现有的命名空间
@@ -184,6 +197,10 @@ class UnifiedSchedulerManager(UnifiedManagerBase):
                     # 兼容旧格式
                     base_url = task_center_url.split('/api/namespaces/')[0]
                     url = f"{base_url}/api/namespaces/{namespace_name}"
+                elif '/namespaces/' in task_center_url:
+                    # 简化格式（无/api前缀），转换为标准格式
+                    base_url = task_center_url.split('/namespaces/')[0]
+                    url = f"{base_url}/api/v1/namespaces/{namespace_name}"
                 elif task_center_url.endswith('/api/v1') or task_center_url.endswith('/api/v1/'):
                     # 多命名空间模式，URL 是 http://localhost:8001/api/v1
                     url = f"{task_center_url.rstrip('/')}/namespaces/{namespace_name}"

jettask/scheduler/models.py

@@ -29,11 +29,15 @@ class TaskStatus(Enum):
 
 @dataclass
 class ScheduledTask:
-    """定时任务模型"""
-    task_name: str                      # 要执行的函数名（对应@app.task注册的函数名）
-    task_type: TaskType                 # 任务类型
-    queue_name: str                     # 目标队列
-    
+    """
+    定时任务模型
+
+    定时任务以 queue 为核心，定期向指定队列发送消息。
+    队列中的消息可以被多个消费者处理，具体处理逻辑由消费者决定。
+    """
+    task_type: TaskType                 # 任务类型（once/interval/cron）
+    queue_name: str                     # 目标队列（必填）
+
     # 可选字段
     id: Optional[int] = None            # 数据库自增ID（唯一标识）
     scheduler_id: Optional[str] = None  # 任务的唯一标识符（用于去重）
@@ -71,14 +75,14 @@ class ScheduledTask:
     def _validate(self):
         """验证任务配置"""
         if self.task_type == TaskType.CRON and not self.cron_expression:
-            raise ValueError(f"Task {self.task_name} with type CRON must have cron_expression")
-        
+            raise ValueError(f"Task (queue={self.queue_name}) with type CRON must have cron_expression")
+
         if self.task_type == TaskType.INTERVAL and not self.interval_seconds:
-            raise ValueError(f"Task {self.task_name} with type INTERVAL must have interval_seconds")
-        
+            raise ValueError(f"Task (queue={self.queue_name}) with type INTERVAL must have interval_seconds")
+
         # ONCE类型任务不应该有interval_seconds参数
         if self.task_type == TaskType.ONCE and self.interval_seconds is not None:
-            raise ValueError(f"Task {self.task_name} with type ONCE should not have interval_seconds. Use next_run_time to specify when to run the task")
+            raise ValueError(f"Task (queue={self.queue_name}) with type ONCE should not have interval_seconds. Use next_run_time to specify when to run the task")
     
     def calculate_next_run_time(self, from_time: Optional[datetime] = None) -> Optional[datetime]:
         """计算下次执行时间"""

jettask/scheduler/schedule.py

@@ -0,0 +1,166 @@
+"""
+定时任务定义类 - 类似 TaskMessage 的设计模式
+用于定义定时任务，然后统一注册到数据库
+"""
+from typing import Optional, Dict, Any, List
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from .models import TaskType
+
+
+@dataclass
+class Schedule:
+    """
+    定时任务定义对象
+
+    这是一个定义定时任务的类，类似 TaskMessage 的设计模式。
+    定时任务会定期向指定队列发送消息，队列中的消息可以被多个消费者处理。
+
+    使用示例:
+        # 1. 定义定时任务
+        schedule1 = Schedule(
+            scheduler_id="notify_every_30s",
+            queue="notification_queue",
+            interval_seconds=30,
+            kwargs={"user_id": "user_123", "message": "定时提醒"}
+        )
+
+        schedule2 = Schedule(
+            scheduler_id="report_cron",
+            queue="report_queue",
+            cron_expression="0 9 * * *",  # 每天9点
+            description="每天生成报告"
+        )
+
+        # 2. 批量注册
+        await app.register_schedules([schedule1, schedule2])
+
+        # 3. 或单个注册
+        await app.register_schedules(schedule1)
+    """
+
+    # 必需参数
+    scheduler_id: str  # 任务唯一标识符（必填）
+    queue: str         # 目标队列（必填）
+
+    # 调度类型（三选一）
+    interval_seconds: Optional[float] = None      # 间隔任务（秒）
+    cron_expression: Optional[str] = None         # Cron表达式任务
+    next_run_time: Optional[datetime] = None      # 一次性任务（指定执行时间）
+
+    # 任务参数
+    args: List[Any] = field(default_factory=list)
+    kwargs: Dict[str, Any] = field(default_factory=dict)
+
+    # 执行选项
+    priority: Optional[int] = None      # 优先级（1最高，数字越大优先级越低）
+    timeout: int = 300                  # 超时时间（秒）
+    max_retries: int = 3                # 最大重试次数
+    retry_delay: int = 60               # 重试延迟（秒）
+
+    # 控制选项
+    enabled: bool = True                # 是否启用
+    description: Optional[str] = None   # 任务描述
+    tags: List[str] = field(default_factory=list)        # 标签
+    metadata: Dict[str, Any] = field(default_factory=dict)  # 额外元数据
+
+    # 注册选项
+    skip_if_exists: bool = True         # 如果已存在则跳过
+    at_once: bool = True                # 是否立即执行一次
+
+    def __post_init__(self):
+        """初始化后处理"""
+        self._validate()
+
+    def _validate(self):
+        """验证配置"""
+        if not self.scheduler_id:
+            raise ValueError("scheduler_id is required")
+
+        if not self.queue:
+            raise ValueError("queue is required")
+
+        # 检查调度类型（必须指定一种）
+        schedule_types = [
+            self.interval_seconds is not None,
+            self.cron_expression is not None,
+            self.next_run_time is not None
+        ]
+
+        if sum(schedule_types) == 0:
+            raise ValueError(
+                "Must specify one schedule type: "
+                "interval_seconds, cron_expression, or next_run_time"
+            )
+
+        if sum(schedule_types) > 1:
+            raise ValueError(
+                "Can only specify one schedule type: "
+                "interval_seconds, cron_expression, or next_run_time"
+            )
+
+        # 验证具体参数
+        if self.interval_seconds is not None and self.interval_seconds <= 0:
+            raise ValueError(f"interval_seconds must be positive, got {self.interval_seconds}")
+
+        if self.priority is not None and self.priority < 1:
+            raise ValueError(f"priority must be positive (1 is highest), got {self.priority}")
+
+    def get_task_type(self) -> TaskType:
+        """获取任务类型"""
+        if self.interval_seconds is not None:
+            return TaskType.INTERVAL
+        elif self.cron_expression is not None:
+            return TaskType.CRON
+        else:
+            return TaskType.ONCE
+
+    def to_dict(self) -> dict:
+        """
+        转换为字典格式（用于注册到数据库）
+
+        Returns:
+            dict: 任务字典
+        """
+        data = {
+            'scheduler_id': self.scheduler_id,
+            'queue': self.queue,
+            'task_type': self.get_task_type().value,
+            'task_args': self.args,
+            'task_kwargs': self.kwargs,
+            'enabled': self.enabled,
+            'priority': self.priority,
+            'timeout': self.timeout,
+            'max_retries': self.max_retries,
+            'retry_delay': self.retry_delay,
+            'description': self.description,
+            'tags': self.tags,
+            'metadata': self.metadata,
+        }
+
+        # 添加调度相关参数
+        if self.interval_seconds is not None:
+            data['interval_seconds'] = self.interval_seconds
+        elif self.cron_expression is not None:
+            data['cron_expression'] = self.cron_expression
+        elif self.next_run_time is not None:
+            data['next_run_time'] = self.next_run_time
+
+        return data
+
+    def __repr__(self) -> str:
+        """友好的字符串表示"""
+        parts = [f"Schedule(scheduler_id='{self.scheduler_id}'"]
+        parts.append(f"queue='{self.queue}'")
+
+        if self.interval_seconds:
+            parts.append(f"interval={self.interval_seconds}s")
+        elif self.cron_expression:
+            parts.append(f"cron='{self.cron_expression}'")
+        elif self.next_run_time:
+            parts.append(f"at={self.next_run_time}")
+
+        if self.kwargs:
+            parts.append(f"kwargs={self.kwargs}")
+
+        return ", ".join(parts) + ")"

jettask/scheduler/scheduler.py

@@ -10,7 +10,7 @@ from typing import Optional, List, TYPE_CHECKING
 from datetime import datetime
 
 from ..utils.task_logger import get_task_logger, LogContext
-from .task_crud import ScheduledTaskManager
+from .database import ScheduledTaskManager
 from .models import ScheduledTask, TaskExecutionHistory, TaskType, TaskStatus as ScheduledTaskStatus
 from .loader import TaskLoader
 
@@ -499,12 +499,12 @@ class TaskScheduler:
             from ..core.message import TaskMessage
             task_messages = []
             for task in bulk_tasks:
-                # 准备kwargs
-                kwargs = task.task_kwargs or {}
-                # 添加任务名称和scheduled_task_id用于路由和跟踪
-                kwargs['__task_name'] = task.task_name
+                # 准备kwargs（合并task定义的kwargs）
+                kwargs = task.task_kwargs.copy() if task.task_kwargs else {}
+
+                # 添加scheduled_task_id用于跟踪
                 kwargs['__scheduled_task_id'] = task.id
-                
+
                 # 将timeout、max_retries等参数放入kwargs中传递
                 if task.timeout:
                     kwargs['__timeout'] = task.timeout
@@ -512,17 +512,18 @@ class TaskScheduler:
                     kwargs['__max_retries'] = task.max_retries
                 if task.retry_delay:
                     kwargs['__retry_delay'] = task.retry_delay
-                
-                # 创建TaskMessage
+
+                # 创建TaskMessage（直接发送到queue，由queue的消费者处理）
                 task_msg = TaskMessage(
                     queue=task.queue_name,
+                    args=task.task_args or [],
                     kwargs=kwargs,
                     priority=task.priority
                 )
                 task_messages.append(task_msg)
             
-            # 使用send_tasks方法发送
-            event_ids = await self.app.send_tasks(task_messages)
+            # 使用send_tasks方法发送（异步模式）
+            event_ids = await self.app.send_tasks(task_messages, asyncio=True)
             logger.info(f"Triggered {len(task_messages)} tasks via send_tasks")
             
             # 准备批量操作的数据
@@ -620,7 +621,7 @@ class TaskScheduler:
         """运行调度器主循环"""
         # 建立Redis连接（使用统一的连接池管理）
         if not self.redis:
-            from jettask.utils.db_connector import get_async_redis_client
+            from jettask.db.connector import get_async_redis_client
 
             self.redis = get_async_redis_client(
                 redis_url=self.redis_url,

jettask/schemas/__init__.py

@@ -57,6 +57,7 @@ from .namespace import (
 
 # 积压监控相关模型
 from .backlog import (
+    BacklogLatestRequest,
     BacklogTrendRequest,
     BacklogSnapshot,
     BacklogStatistics,
@@ -91,6 +92,31 @@ from .monitoring import (
     WorkerMetrics
 )
 
+# API 响应模型
+from .responses import (
+    SuccessResponse,
+    DataResponse,
+    PaginatedDataResponse,
+    SystemStatsData,
+    SystemStatsResponse,
+    DashboardStatsData,
+    DashboardStatsResponse,
+    QueueRankingItem,
+    TopQueuesResponse,
+    NamespaceStatistics,
+    NamespaceStatisticsResponse,
+    WorkerHeartbeatInfo,
+    WorkersResponse,
+    WorkerSummary,
+    WorkerSummaryResponse,
+    WorkerOfflineRecord,
+    WorkerOfflineHistoryResponse,
+    DatabaseStatus,
+    SystemSettings,
+    SystemSettingsResponse,
+    DatabaseStatusResponse
+)
+
 __all__ = [
     # Task
     'TasksRequest',
@@ -162,5 +188,28 @@ __all__ = [
     'SystemHealth',
     'DashboardOverviewRequest',
     'DashboardOverview',
-    'WorkerMetrics'
+    'WorkerMetrics',
+
+    # Responses
+    'SuccessResponse',
+    'DataResponse',
+    'PaginatedDataResponse',
+    'SystemStatsData',
+    'SystemStatsResponse',
+    'DashboardStatsData',
+    'DashboardStatsResponse',
+    'QueueRankingItem',
+    'TopQueuesResponse',
+    'NamespaceStatistics',
+    'NamespaceStatisticsResponse',
+    'WorkerHeartbeatInfo',
+    'WorkersResponse',
+    'WorkerSummary',
+    'WorkerSummaryResponse',
+    'WorkerOfflineRecord',
+    'WorkerOfflineHistoryResponse',
+    'DatabaseStatus',
+    'SystemSettings',
+    'SystemSettingsResponse',
+    'DatabaseStatusResponse'
 ]