jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl

jettask/core/consumer_manager.py

@@ -155,7 +155,7 @@ class ConsumerManager:
             # 由于已经在_validate_strategy_configuration中验证过，
             # 这里应该只会在MainProcess中执行
             self._consumer_name = pod_name
-            logger.info(f"使用Pod策略的consumer名称: {self._consumer_name}")
+            logger.debug(f"使用Pod策略的consumer名称: {self._consumer_name}")
                 
         return f"{self._consumer_name}-{queue}"
     
@@ -195,6 +195,16 @@ class ConsumerManager:
             return self._heartbeat_strategy.is_heartbeat_timeout()
         return False
     
+    def record_group_info(self, queue: str, task_name: str, group_name: str, consumer_name: str):
+        """记录task的group信息到worker hash表（仅对HEARTBEAT策略有效）"""
+        if self.strategy == ConsumerStrategy.HEARTBEAT and self._heartbeat_strategy:
+            self._heartbeat_strategy.record_group_info(queue, task_name, group_name, consumer_name)
+    
+    async def record_group_info_async(self, queue: str, task_name: str, group_name: str, consumer_name: str):
+        """异步记录task的group信息到worker hash表（仅对HEARTBEAT策略有效）"""
+        if self.strategy == ConsumerStrategy.HEARTBEAT and self._heartbeat_strategy:
+            await self._heartbeat_strategy.record_group_info_async(queue, task_name, group_name, consumer_name)
+    
     def cleanup_expired_consumers(self, queue: str):
         """清理过期的消费者（可选功能）"""
         try:
@@ -380,10 +390,10 @@ class HeartbeatConsumerStrategy:
                     break
             
             if not worker_keys:
-                logger.info("No worker keys found during scan")
+                logger.debug("No worker keys found during scan")
                 return None
             else:
-                logger.info(f"Found {len(worker_keys)} worker keys to check")
+                logger.debug(f"Found {len(worker_keys)} worker keys to check")
             
             # 查找符合条件的离线worker
             offline_workers = []
@@ -392,7 +402,7 @@ class HeartbeatConsumerStrategy:
                 try:
                     # 获取worker数据
                     worker_data = self.redis.hgetall(worker_key)
-                    # logger.info(f'{worker_key=} {worker_data=}')
+                    # logger.debug(f'{worker_key=} {worker_data=}')
                     if not worker_data:
                         continue
                     # 获取worker的状态信息
@@ -420,15 +430,15 @@ class HeartbeatConsumerStrategy:
                     # 1. is_alive标记为false，或者
                     # 2. 最后心跳时间超过了heartbeat_timeout
                     is_truly_offline = (not is_alive) or (current_time - last_heartbeat > self.heartbeat_timeout)
-                    # logger.info(f'{is_truly_offline=} {worker_data=}')
+                    # logger.debug(f'{is_truly_offline=} {worker_data=}')
                     if not is_truly_offline:
-                        logger.info(f"Worker {is_alive=} {current_time - last_heartbeat} {self.heartbeat_timeout} {worker_data.get('consumer_id')} is still active (last_heartbeat: {current_time - last_heartbeat:.1f}s ago)")
+                        logger.debug(f"Worker {is_alive=} {current_time - last_heartbeat} {self.heartbeat_timeout} {worker_data.get('consumer_id')} is still active (last_heartbeat: {current_time - last_heartbeat:.1f}s ago)")
                         continue
                     
                     # 需要离线超过heartbeat_timeout才能被复用（与离线检测保持一致）
                     # min_offline_duration = self.heartbeat_timeout
                     # if offline_time > 0 and (current_time - offline_time) < min_offline_duration:
-                    #     logger.info(f"Worker {worker_data.get('consumer_id')} offline for only {current_time - offline_time:.1f}s, need {min_offline_duration}s")
+                    #     logger.debug(f"Worker {worker_data.get('consumer_id')} offline for only {current_time - offline_time:.1f}s, need {min_offline_duration}s")
                     #     continue
                     
                     # 获取consumer_id
@@ -484,7 +494,7 @@ class HeartbeatConsumerStrategy:
             
             pipeline.execute()
             
-            logger.info(f"Found reusable worker: {selected_consumer_id}, offline since {time.time() - selected_offline_time:.1f}s ago")
+            logger.debug(f"Found reusable worker: {selected_consumer_id}, offline since {time.time() - selected_offline_time:.1f}s ago")
             return selected_consumer_id
             
         except Exception as e:
@@ -734,9 +744,9 @@ class HeartbeatConsumerStrategy:
             if not self.consumer_id:
                 # 如果没有可复用的，生成新的consumer ID
                 self.consumer_id = f"{self.hostname_prefix}-{uuid.uuid4().hex[:8]}-{os.getpid()}"
-                logger.info(f"Created new consumer ID: {self.consumer_id}")
+                logger.debug(f"Created new consumer ID: {self.consumer_id}")
             else:
-                logger.info(f"Reusing offline worker ID: {self.consumer_id}")
+                logger.debug(f"Reusing offline worker ID: {self.consumer_id}")
             
             # 更新worker_key
             self._worker_key = f'{self.redis_prefix}:{self.worker_prefix}:{self.consumer_id}'
@@ -771,9 +781,91 @@ class HeartbeatConsumerStrategy:
             if queue not in self._heartbeat_processes:
                 self._start_heartbeat_process_for_queue(queue)
             
-            logger.info(f"Created consumer name for queue {queue}: {self.consumer_names[queue]}")
+            logger.debug(f"Created consumer name for queue {queue}: {self.consumer_names[queue]}")
         return self.consumer_names[queue]
     
+    def record_group_info(self, queue: str, task_name: str, group_name: str, consumer_name: str):
+        """记录task的group信息到worker hash表
+        
+        Args:
+            queue: 队列名
+            task_name: 任务名
+            group_name: consumer group名称
+            consumer_name: consumer名称
+        """
+        try:
+            # 确保worker_key已初始化
+            if not self._worker_key:
+                self._ensure_consumer_id()
+                if not self._worker_key:
+                    logger.warning("Cannot record group info: worker_key not initialized")
+                    return
+            
+            # 构建group信息
+            import json
+            group_info = {
+                'queue': queue,
+                'task_name': task_name,
+                'group_name': group_name,
+                'consumer_name': consumer_name,
+                'stream_key': f"{self.redis_prefix}:QUEUE:{queue}"
+            }
+            
+            # 将group信息存储到worker的hash中
+            # 使用 group_info:{group_name} 作为field
+            field_name = f"group_info:{group_name}"
+            self.redis.hset(
+                self._worker_key,
+                field_name,
+                json.dumps(group_info)
+            )
+            
+            logger.debug(f"Recorded group info for task {task_name}: {group_info}")
+            
+        except Exception as e:
+            logger.error(f"Error recording task group info: {e}")
+    
+    async def record_group_info_async(self, queue: str, task_name: str, group_name: str, consumer_name: str):
+        """异步记录task的group信息到worker hash表
+        
+        Args:
+            queue: 队列名
+            task_name: 任务名
+            group_name: consumer group名称
+            consumer_name: consumer名称
+        """
+        try:
+            # 确保worker_key已初始化
+            if not self._worker_key:
+                self._ensure_consumer_id()
+                if not self._worker_key:
+                    logger.warning("Cannot record group info: worker_key not initialized")
+                    return
+            
+            # 构建group信息
+            import json
+            group_info = {
+                'queue': queue,
+                'task_name': task_name,
+                'group_name': group_name,
+                'consumer_name': consumer_name,
+                'stream_key': f"{self.redis_prefix}:QUEUE:{queue}"
+            }
+            
+            # 将group信息存储到worker的hash中
+            # 使用 group_info:{group_name} 作为field
+            field_name = f"group_info:{group_name}"
+            await self.async_redis.hset(
+                self._worker_key,
+                field_name,
+                json.dumps(group_info)
+            )
+            
+            logger.debug(f"Recorded group info for task {task_name}: {group_info}")
+            
+        except Exception as e:
+            logger.error(f"Error recording task group info: {e}")
+    
     def _ensure_worker_initialized(self):
         """确保worker已初始化"""
         if self.consumer_id is None:
@@ -788,7 +880,7 @@ class HeartbeatConsumerStrategy:
             # 心跳进程已经在运行，只需要记录这个队列
             self._heartbeat_processes[queue] = True
             return
-        logger.info('启动心跳进程')
+        logger.debug('启动心跳进程')
         # 第一次调用时创建心跳进程管理器
         if self._heartbeat_process_manager is None:
             # 获取Redis URL
@@ -845,11 +937,11 @@ class HeartbeatConsumerStrategy:
         self.redis.hset(self._worker_key, mapping=worker_info)
         # 同时添加到 sorted set
         self.redis.zadd(f"{self.redis_prefix}:ACTIVE_WORKERS", {self.consumer_id: current_time})
-        logger.info(f"Initialized worker {self.consumer_id} with key {self._worker_key}")
+        logger.debug(f"Initialized worker {self.consumer_id} with key {self._worker_key}")
         
         self._heartbeat_process_manager.add_queue(queue, self._worker_key)
         self._heartbeat_processes[queue] = True
-        # logger.info(f"Started heartbeat process for queue {queue}")
+        # logger.debug(f"Started heartbeat process for queue {queue}")
     
     def _start_scanner(self):
         """启动扫描器协程"""
@@ -858,7 +950,7 @@ class HeartbeatConsumerStrategy:
             self._scanner_task = loop.create_task(self._scanner_loop())
             # 立即执行一次扫描，清理可能存在的死亡worker
             loop.create_task(self._immediate_scan())
-            # logger.info("Started heartbeat scanner coroutine")
+            # logger.debug("Started heartbeat scanner coroutine")
         except RuntimeError:
             # 没有运行中的事件循环，标记为需要启动
             logger.debug("No running event loop, scanner will be started when async context is available")
@@ -867,9 +959,9 @@ class HeartbeatConsumerStrategy:
     async def _immediate_scan(self):
         """启动时立即执行一次扫描（协程版本）"""
         try:
-            # logger.info("Performing immediate scan for dead workers...")
+            # logger.debug("Performing immediate scan for dead workers...")
             await self._perform_scan()
-            # logger.info("Immediate scan completed")
+            # logger.debug("Immediate scan completed")
         except Exception as e:
             logger.error(f"Error in immediate scan: {e}")
     
@@ -955,7 +1047,7 @@ class HeartbeatConsumerStrategy:
                     
                     if is_alive and last_heartbeat < worker_timeout_threshold:
                         # 心跳超时的活跃worker
-                        logger.info(f"Worker {consumer_id} timeout detected: "
+                        logger.debug(f"Worker {consumer_id} timeout detected: "
                                   f"last_heartbeat={last_heartbeat}, "
                                   f"timeout={worker_heartbeat_timeout}s, "
                                   f"threshold={worker_timeout_threshold}")
@@ -969,7 +1061,7 @@ class HeartbeatConsumerStrategy:
                     continue
             
             if timeout_workers:
-                logger.info(f"Found {len(timeout_workers)} timeout workers")
+                logger.debug(f"Found {len(timeout_workers)} timeout workers")
                 
                 for worker_key, worker_data in timeout_workers:
                     consumer_id = worker_data.get('consumer_id')
@@ -996,10 +1088,10 @@ class HeartbeatConsumerStrategy:
                         # 再次检查worker是否真的超时（避免竞态条件）
                         current_heartbeat = await self.async_redis.hget(worker_key, 'last_heartbeat')
                         if current_heartbeat and float(current_heartbeat) >= timeout_threshold:
-                            logger.info(f"Worker {consumer_id} is now alive, skipping")
+                            logger.debug(f"Worker {consumer_id} is now alive, skipping")
                             continue
                         
-                        logger.info(f"Processing timeout worker: {consumer_id}")
+                        logger.debug(f"Processing timeout worker: {consumer_id}")
                         # 只标记worker为离线
                         await self._mark_worker_offline(worker_key, worker_data)
                         
@@ -1029,7 +1121,7 @@ class HeartbeatConsumerStrategy:
                     'shutdown_reason': 'heartbeat_timeout',
                     'messages_transferred': 'false'  # 初始状态：消息未转移
                 })
-                logger.info(f"Marked worker {consumer_id} as offline with messages_transferred=false")
+                logger.debug(f"Marked worker {consumer_id} as offline with messages_transferred=false")
             else:
                 # 已经是离线状态的worker，只更新离线时间
                 await self.async_redis.hset(worker_key, 'offline_time', str(current_time))
@@ -1055,7 +1147,7 @@ class HeartbeatConsumerStrategy:
         try:
             loop = asyncio.get_running_loop()
             self._stats_flusher_task = loop.create_task(self._stats_flusher_loop())
-            logger.info("Started stats flusher coroutine")
+            logger.debug("Started stats flusher coroutine")
         except RuntimeError:
             # 没有运行中的事件循环，标记为需要启动
             logger.debug("No running event loop for stats flusher, will be started when async context is available")
@@ -1085,7 +1177,7 @@ class HeartbeatConsumerStrategy:
             prefixed_queue = self.get_prefixed_queue_name(queue)
             result = self.redis.execute_command('XGROUP', 'DELCONSUMER', prefixed_queue, prefixed_queue, consumer_name)
             if result > 0:
-                logger.info(f"Deleted stream consumer {consumer_name} from group {queue}")
+                logger.debug(f"Deleted stream consumer {consumer_name} from group {queue}")
             else:
                 logger.debug(f"Stream consumer {consumer_name} was not found in group {queue}")
         except Exception as e:
@@ -1118,13 +1210,13 @@ class HeartbeatConsumerStrategy:
             # 再次检查worker是否真的超时（避免竞态条件）
             current_score = await self.async_redis.zscore(heartbeat_key, worker_data)
             if current_score and time.time() - current_score < self.heartbeat_timeout:
-                logger.info(f"Worker {consumer_name} is now alive, skipping")
+                logger.debug(f"Worker {consumer_name} is now alive, skipping")
                 return
             
             # 从有序集合中删除死亡的worker（使用原始的worker_data）
             removed = await self.async_redis.zrem(heartbeat_key, worker_data)
             if removed:
-                logger.info(f"Removed dead worker {consumer_name} from heartbeat set for queue {queue}")
+                logger.debug(f"Removed dead worker {consumer_name} from heartbeat set for queue {queue}")
                 
                 # 重置该consumer的pending消息
                 await self._reset_consumer_pending_messages(queue, consumer_name)
@@ -1201,7 +1293,7 @@ class HeartbeatConsumerStrategy:
                         pass
                     return
                 
-                logger.info(f"Found {len(consumer_messages)} pending messages for dead consumer {consumer_name}")
+                logger.debug(f"Found {len(consumer_messages)} pending messages for dead consumer {consumer_name}")
                 
                 # 获取消息ID列表
                 message_ids = [msg['message_id'] for msg in consumer_messages]
@@ -1295,7 +1387,7 @@ class HeartbeatConsumerStrategy:
                     'status': 'completed' if not failed_messages else 'completed_with_errors'
                 })
                 
-                logger.info(f"Recovery completed: {recovered_count}/{len(message_ids)} messages recovered from {consumer_name}")
+                logger.debug(f"Recovery completed: {recovered_count}/{len(message_ids)} messages recovered from {consumer_name}")
                 
                 if failed_messages:
                     logger.error(f"Failed to recover {len(failed_messages)} messages: {failed_messages[:10]}...")
@@ -1463,7 +1555,7 @@ class HeartbeatConsumerStrategy:
                                     # 没有pending消息，直接删除
                                     try:
                                         await self.async_redis.execute_command('XGROUP', 'DELCONSUMER', queue, queue, consumer_name)
-                                        logger.info(f"Cleaned up stale recovery consumer {consumer_name}")
+                                        logger.debug(f"Cleaned up stale recovery consumer {consumer_name}")
                                         cleaned_count += 1
                                     except Exception as e:
                                         logger.error(f"Failed to delete recovery consumer {consumer_name}: {e}")
@@ -1474,7 +1566,7 @@ class HeartbeatConsumerStrategy:
                     logger.error(f"Traceback:\n{traceback.format_exc()}")
             
             if cleaned_count > 0:
-                logger.info(f"Cleaned up {cleaned_count} stale recovery consumers")
+                logger.debug(f"Cleaned up {cleaned_count} stale recovery consumers")
                 
         except Exception as e:
             logger.error(f"Error in cleanup_stale_recovery_consumers: {e}")
@@ -1548,7 +1640,7 @@ class HeartbeatConsumerStrategy:
             
             # 如果worker从未运行过（没有数据），则不需要处理
             if not worker_data:
-                logger.info(f"Worker {self.consumer_id} never started, skipping cleanup")
+                logger.debug(f"Worker {self.consumer_id} never started, skipping cleanup")
                 return
             
             # 更新worker状态为离线（保留所有现有数据）
@@ -1573,14 +1665,14 @@ class HeartbeatConsumerStrategy:
             
             # 不再保存历史记录，WORKER键本身就包含了所有信息
             
-            logger.info(f"Marked worker {self.consumer_id} as offline immediately")
+            logger.debug(f"Marked worker {self.consumer_id} as offline immediately")
             
         except Exception as e:
             logger.error(f"Failed to mark worker as offline during cleanup: {e}")
         
         # 如果从未成功运行过，直接返回
         if not worker_data:
-            logger.info(f"Heartbeat consumer {self.consumer_id} stopped gracefully (never started)")
+            logger.debug(f"Heartbeat consumer {self.consumer_id} stopped gracefully (never started)")
             return
         
         # 等待扫描线程结束（非阻塞）
@@ -1600,4 +1692,4 @@ class HeartbeatConsumerStrategy:
         # 重要：不删除心跳记录！
         # 心跳记录必须保留，让scanner能够检测到worker离线并恢复pending消息
         # 心跳会因为超时自动被scanner清理
-        logger.info(f"Heartbeat consumer {self.consumer_id} stopped")
+        logger.debug(f"Heartbeat consumer {self.consumer_id} stopped")

jettask/core/context.py

@@ -20,6 +20,8 @@ class TaskContext:
     async def my_task(ctx: TaskContext, data: dict):
         print(f"Task ID: {ctx.event_id}")
         print(f"Task Name: {ctx.name}")
+        if ctx.scheduled_task_id:
+            print(f"Triggered by scheduled task: {ctx.scheduled_task_id}")
         return data
     """
     event_id: str
@@ -29,6 +31,7 @@ class TaskContext:
     queue: Optional[str] = None
     worker_id: Optional[str] = None
     retry_count: int = 0
+    scheduled_task_id: Optional[int] = None  # 定时任务ID（如果由定时任务触发）
     
     def __repr__(self) -> str:
         return f"TaskContext(event_id={self.event_id}, name={self.name}, queue={self.queue})"

jettask/core/enums.py

@@ -0,0 +1,137 @@
+"""
+任务状态和结果定义
+"""
+from enum import Enum
+from dataclasses import dataclass
+from typing import Any, Optional, Dict
+import time
+
+
+class TaskStatus(Enum):
+    """任务状态枚举"""
+    # 基本状态
+    PENDING = "pending"          # 任务已创建，等待执行
+    RUNNING = "running"          # 任务正在执行
+    SUCCESS = "success"          # 任务执行成功
+    ERROR = "error"              # 任务执行失败
+    
+    # 特殊状态
+    DELAYED = "delayed"          # 延迟任务，等待触发
+    REJECTED = "rejected"        # 任务被拒绝（on_before返回reject）
+    RETRY = "retry"              # 任务等待重试
+    TIMEOUT = "timeout"          # 任务执行超时
+    CANCELLED = "cancelled"      # 任务被取消
+    
+    # 定时任务相关状态
+    SCHEDULED = "scheduled"      # 已调度（定时任务专用）
+    
+    @classmethod
+    def is_terminal(cls, status: 'TaskStatus') -> bool:
+        """判断是否是终态（不会再改变的状态）"""
+        return status in {cls.SUCCESS, cls.ERROR, cls.REJECTED, cls.TIMEOUT, cls.CANCELLED}
+    
+    @classmethod
+    def is_active(cls, status: 'TaskStatus') -> bool:
+        """判断是否是活跃状态（正在处理中）"""
+        return status in {cls.PENDING, cls.RUNNING, cls.DELAYED, cls.RETRY, cls.SCHEDULED}
+
+
+@dataclass
+class TaskResult:
+    """
+    任务结果对象
+    apply_async 返回的结果，包含任务的完整信息
+    """
+    id: str                                    # 任务ID（event_id）
+    name: str                                   # 任务名称
+    queue: str                                  # 队列名称
+    status: TaskStatus = TaskStatus.PENDING    # 任务状态
+    created_at: float = None                   # 创建时间
+    trigger_time: float = None                 # 触发时间
+    scheduled_task_id: Optional[int] = None    # 定时任务ID（如果由定时任务触发）
+    args: tuple = None                         # 位置参数
+    kwargs: dict = None                        # 关键字参数
+    metadata: Dict[str, Any] = None           # 其他元数据
+    
+    def __post_init__(self):
+        """初始化默认值"""
+        if self.created_at is None:
+            self.created_at = time.time()
+        if self.trigger_time is None:
+            self.trigger_time = self.created_at
+        if self.args is None:
+            self.args = ()
+        if self.kwargs is None:
+            self.kwargs = {}
+        if self.metadata is None:
+            self.metadata = {}
+    
+    def __str__(self) -> str:
+        """字符串表示"""
+        return f"TaskResult(id={self.id}, name={self.name}, status={self.status.value})"
+    
+    def __repr__(self) -> str:
+        """详细表示"""
+        return (f"TaskResult(id={self.id}, name={self.name}, queue={self.queue}, "
+                f"status={self.status.value}, created_at={self.created_at})")
+    
+    async def wait(self, timeout: Optional[float] = None):
+        """
+        等待任务完成
+        
+        Args:
+            timeout: 超时时间（秒）
+            
+        Returns:
+            任务执行结果
+        """
+        # TODO: 实现等待逻辑
+        raise NotImplementedError("wait method not implemented yet")
+    
+    async def get_result(self, timeout: Optional[float] = None):
+        """
+        获取任务结果
+        
+        Args:
+            timeout: 超时时间（秒）
+            
+        Returns:
+            任务执行结果
+        """
+        # TODO: 实现获取结果逻辑
+        raise NotImplementedError("get_result method not implemented yet")
+    
+    async def cancel(self) -> bool:
+        """
+        取消任务
+        
+        Returns:
+            是否成功取消
+        """
+        # TODO: 实现取消逻辑
+        raise NotImplementedError("cancel method not implemented yet")
+    
+    async def get_status(self) -> TaskStatus:
+        """
+        获取任务当前状态
+        
+        Returns:
+            任务状态
+        """
+        # TODO: 实现获取状态逻辑
+        raise NotImplementedError("get_status method not implemented yet")
+    
+    @property
+    def is_ready(self) -> bool:
+        """任务是否已完成（终态）"""
+        return TaskStatus.is_terminal(self.status)
+    
+    @property
+    def is_successful(self) -> bool:
+        """任务是否成功"""
+        return self.status == TaskStatus.SUCCESS
+    
+    @property
+    def is_failed(self) -> bool:
+        """任务是否失败"""
+        return self.status in {TaskStatus.ERROR, TaskStatus.TIMEOUT, TaskStatus.REJECTED}