jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl

jettask/core/event_pool.py

@@ -2,7 +2,6 @@ from ..utils.serializer import dumps_str, loads_str
 import time
 import threading
 import logging
-import contextlib
 import asyncio
 from collections import defaultdict, deque, Counter
 from typing import List, Optional, TYPE_CHECKING, Union
@@ -68,6 +67,12 @@ class EventPool(object):
         # 创建带前缀的队列名称映射
         self.prefixed_queues = {}
         
+        # 优先级队列管理
+        self.priority_queues = {}  # {base_queue: [queue:1, queue:2, ...]}
+        self.priority_queues_lock = threading.Lock()
+        self._last_priority_scan = {}  # 记录上次扫描时间
+        self._priority_scan_interval = 5  # 5秒扫描一次，及时发现新的优先级队列
+        
         # 用于跟踪广播消息
         self._broadcast_message_tracker = {}
         
@@ -117,6 +122,7 @@ class EventPool(object):
         """为队列名称添加前缀"""
         return f"{self.redis_prefix}:QUEUE:{queue}"
     
+    
     def get_redis_client(self, asyncio: bool = False, binary: bool = False):
         """获取Redis客户端
         
@@ -128,84 +134,62 @@ class EventPool(object):
             return self.async_binary_redis_client if asyncio else self.binary_redis_client
         return self.async_redis_client if asyncio else self.redis_client
 
-    def create_group(self):
-        """创建消费组 - 现在consumer group在listen_event_by_task中动态创建"""
-        for queue in self.queues:
-            prefixed_queue = self.get_prefixed_queue_name(queue)
-            
-            # 创建默认消费组（用于兼容性）
-            with contextlib.suppress(Exception):
-                prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-                self.binary_redis_client.xgroup_create(
-                    name=prefixed_queue_bytes,
-                    groupname=prefixed_queue_bytes,
-                    id=b"0",
-                    mkstream=True,
-                )
-
-    def send_event(self, queue, message: dict, asyncio: bool = False):
-        # 使用二进制客户端进行Stream操作
-        client = self.get_redis_client(asyncio, binary=True)
-        prefixed_queue = self.get_prefixed_queue_name(queue)
-        # 确保键是bytes类型，但值不需要编码（msgpack已经返回二进制）
-        prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-        stream_data = {b'data': dumps_str(message)}  # 使用bytes键
-        try:
-            event_id = client.xadd(prefixed_queue_bytes, stream_data)
-            # 将bytes类型的event_id转换为字符串返回
-            if isinstance(event_id, bytes):
-                event_id = event_id.decode('utf-8')
-            return event_id
-        except redis.exceptions.ResponseError as e:
-            # 如果队列不存在，创建它
-            if "ERR" in str(e):
-                logger.warning(f'队列 {prefixed_queue} 不存在，正在创建...')
-                try:
-                    # 先创建队列
-                    event_id = client.xadd(prefixed_queue_bytes, stream_data)
-                    # 再创建消费者组
-                    with contextlib.suppress(Exception):
-                        client.xgroup_create(
-                            name=prefixed_queue_bytes,
-                            groupname=prefixed_queue_bytes,
-                            id=b"0"
-                        )
-                    # 将bytes类型的event_id转换为字符串返回
-                    if isinstance(event_id, bytes):
-                        event_id = event_id.decode('utf-8')
-                    return event_id
-                except Exception as create_error:
-                    logger.error(f'创建队列失败: {create_error}')
-                    raise
-            else:
-                raise
-
-    def batch_send_event(self, queue, messages: List[dict], asyncio: bool = False):
-        # 使用二进制客户端进行Stream操作
-        client = self.get_redis_client(asyncio, binary=True)
-        pipe = client.pipeline()
-        prefixed_queue = self.get_prefixed_queue_name(queue)
-        prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-        if asyncio:
-            return self._batch_send_event(prefixed_queue_bytes, messages, pipe)
-        for message in messages:
-            # 确保消息格式正确
-            if 'data' in message:
-                binary_message = {b'data': message['data'] if isinstance(message['data'], bytes) else dumps_str(message['data'])}
-            else:
-                binary_message = message
-            pipe.xadd(prefixed_queue_bytes, binary_message)
-        return pipe.execute()
 
     async def _batch_send_event(self, prefixed_queue, messages: List[dict], pipe):
+        # 使用Lua脚本批量发送消息并添加自增offset
+        lua_script = """
+        local stream_key = KEYS[1]
+        local prefix = ARGV[1]
+        local results = {}
+        
+        -- 使用Hash存储所有队列的offset
+        local offsets_hash = prefix .. ':QUEUE_OFFSETS'
+        
+        -- 从stream_key中提取队列名（去掉prefix:QUEUE:前缀）
+        local queue_name = string.gsub(stream_key, '^' .. prefix .. ':QUEUE:', '')
+        
+        -- 从ARGV[2]开始，每个参数是一个消息的data
+        for i = 2, #ARGV do
+            local data = ARGV[i]
+            
+            -- 使用HINCRBY原子递增offset（如果不存在会自动创建并设为1）
+            local current_offset = redis.call('HINCRBY', offsets_hash, queue_name, 1)
+            
+            -- 添加消息到Stream（包含offset字段）
+            local stream_id = redis.call('XADD', stream_key, '*', 
+                'data', data,
+                'offset', current_offset)
+            
+            table.insert(results, stream_id)
+        end
+        
+        return results
+        """
+        
+        # 准备Lua脚本参数
+        lua_args = [self.redis_prefix.encode() if isinstance(self.redis_prefix, str) else self.redis_prefix]
+        
         for message in messages:
             # 确保消息格式正确
             if 'data' in message:
-                binary_message = {b'data': message['data'] if isinstance(message['data'], bytes) else dumps_str(message['data'])}
+                data = message['data'] if isinstance(message['data'], bytes) else dumps_str(message['data'])
             else:
-                binary_message = message
-            await pipe.xadd(prefixed_queue, binary_message)
-        return await pipe.execute()
+                data = dumps_str(message)
+            lua_args.append(data)
+        
+        # 获取异步Redis客户端（不使用pipe，直接使用client）
+        client = self.get_redis_client(asyncio=True, binary=True)
+        
+        # 执行Lua脚本
+        results = await client.eval(
+            lua_script,
+            1,  # 1个KEY
+            prefixed_queue,  # KEY[1]: stream key
+            *lua_args  # ARGV: prefix, data1, data2, ...
+        )
+        
+        # 解码所有返回的Stream ID
+        return [r.decode('utf-8') if isinstance(r, bytes) else r for r in results]
     
     def is_urgent(self, routing_key):
         is_urgent = self.solo_urgent_retry.get(routing_key, False)
@@ -213,6 +197,107 @@ class EventPool(object):
             del self.solo_urgent_retry[routing_key]
         return is_urgent
     
+    async def scan_priority_queues(self, base_queue: str) -> list:
+        """扫描Redis中的优先级队列
+        
+        Args:
+            base_queue: 基础队列名（不带优先级后缀）
+        
+        Returns:
+            按优先级排序的队列列表
+        """
+        pattern = f"{self.redis_prefix}:QUEUE:{base_queue}:*"
+        
+        try:
+            # 使用SCAN命令扫描匹配的键
+            cursor = 0
+            priority_queues = set()
+            
+            while True:
+                cursor, keys = await self.async_binary_redis_client.scan(
+                    cursor=cursor,
+                    match=pattern.encode() if isinstance(pattern, str) else pattern,
+                    count=100
+                )
+                
+                for key in keys:
+                    # 解析键名获取优先级
+                    key_str = key.decode() if isinstance(key, bytes) else key
+                    # 提取优先级后缀
+                    parts = key_str.split(':')
+                    if len(parts) >= 4:  # jettask:QUEUE:base_queue:priority
+                        queue_with_priority = ':'.join(parts[2:])  # base_queue:priority
+                        priority_queues.add(queue_with_priority)
+                
+                if cursor == 0:
+                    break
+            
+            # 添加基础队列（无优先级）
+            priority_queues.add(base_queue)
+            
+            # 按优先级排序（数字越小优先级越高）
+            sorted_queues = []
+            for q in priority_queues:
+                if ':' in q:
+                    base, priority = q.rsplit(':', 1)
+                    if base == base_queue and priority.isdigit():
+                        sorted_queues.append((int(priority), q))
+                    else:
+                        sorted_queues.append((float('inf'), q))  # 非数字优先级放最后
+                else:
+                    sorted_queues.append((float('inf'), q))  # 无优先级放最后
+            
+            sorted_queues.sort(key=lambda x: x[0])
+            return [q[1] for q in sorted_queues]
+            
+        except Exception as e:
+            logger.error(f"Error scanning priority queues for {base_queue}: {e}")
+            return [base_queue]  # 返回基础队列作为fallback
+    
+    async def update_priority_queues_cache(self, base_queue: str):
+        """更新优先级队列缓存
+        
+        Args:
+            base_queue: 基础队列名
+        """
+        current_time = time.time()
+        
+        # 检查是否需要重新扫描
+        last_scan = self._last_priority_scan.get(base_queue, 0)
+        if current_time - last_scan < self._priority_scan_interval:
+            return  # 未到扫描时间
+        
+        # 扫描并更新缓存
+        priority_queues = await self.scan_priority_queues(base_queue)
+        
+        with self.priority_queues_lock:
+            self.priority_queues[base_queue] = priority_queues
+            self._last_priority_scan[base_queue] = current_time
+        
+        # if len(priority_queues) > 1:
+        #     logger.info(f"Updated priority queues for {base_queue}: {priority_queues}")
+    
+    def get_priority_queues(self, base_queue: str) -> list:
+        """获取优先级队列列表（从缓存）
+        
+        Args:
+            base_queue: 基础队列名
+        
+        Returns:
+            优先级队列列表（已加上前缀）
+        """
+        with self.priority_queues_lock:
+            queues = self.priority_queues.get(base_queue, [])
+            # 返回已加上前缀的队列名
+            result = []
+            for q in queues:
+                if q == base_queue:
+                    continue  # 跳过基础队列，在listen_event_by_task中已经处理
+                # 只返回优先级队列（带优先级后缀的）
+                if ':' in q and q.rsplit(':', 1)[1].isdigit():
+                    result.append(f"{self.redis_prefix}:QUEUE:{q}")
+            return result
+    
     @classmethod
     def separate_by_key(cls, lst):
         groups = {}
@@ -456,7 +541,7 @@ class EventPool(object):
             consecutive_errors += 1
             return False, consecutive_errors
 
-    def _process_message_common(self, event_id: str, event_data: dict, queue: str, event_queue, is_async: bool = False, consumer_name: str = None):
+    def _process_message_common(self, event_id: str, event_data: dict, queue: str, event_queue, is_async: bool = False, consumer_name: str = None, group_name: str = None):
         """通用的消息处理逻辑，供同步和异步版本使用"""
         # 检查消息是否已被认领，防止重复处理
         if event_id in self._claimed_message_ids:
@@ -491,18 +576,17 @@ class EventPool(object):
         
         routing = final_event_data.get("routing")
         
-        # 从consumer_name中提取group_name
-        # consumer_name格式: "YYDG-xxx:task_name"  
-        # group_name格式: "prefixed_queue:task_name"
-        group_name = None
-        if consumer_name and ':' in consumer_name:
-            # 从consumer_name中提取task_name
-            task_name_part = consumer_name.split(':', 1)[1]
+        # 从消息体中获取实际的队列名（可能包含优先级后缀）
+        # 这确保ACK使用正确的stream key
+        actual_queue = final_event_data.get('queue', queue)
+        
+        # 如果没有传入group_name，使用默认值（prefixed_queue）
+        if not group_name:
             prefixed_queue = self.get_prefixed_queue_name(queue)
-            group_name = f"{prefixed_queue}:{task_name_part}"
+            group_name = prefixed_queue
         
         task_item = {
-            "queue": queue,
+            "queue": actual_queue,  # 使用消息体中的实际队列名（可能包含优先级）
             "event_id": actual_event_id,
             "event_data": final_event_data,  # 使用解析后的数据
             "consumer": consumer_name,  # 添加消费者信息
@@ -573,11 +657,19 @@ class EventPool(object):
         )
         
         # 等待consumer manager初始化
+        # 对于优先级队列，使用基础队列名来获取consumer
+        base_queue = queue
+        if ':' in queue and queue.rsplit(':', 1)[-1].isdigit():
+            base_queue = queue.rsplit(':', 1)[0]
+        
         wait_times = [0.1, 0.2, 0.4, 0.8, 1.6, 3.2]
         for wait_time in wait_times:
             try:
-                current_consumer = self.consumer_manager.get_consumer_name(queue)
+                current_consumer = self.consumer_manager.get_consumer_name(base_queue)
                 if current_consumer:
+                    # 对于优先级队列，consumer名称需要添加队列后缀
+                    if base_queue != queue:
+                        current_consumer = f"{current_consumer}:{queue.rsplit(':', 1)[-1]}"
                     logger.info(f"Consumer manager initialized for queue {queue}, consumer: {current_consumer}")
                     break
             except Exception as e:
@@ -587,19 +679,18 @@ class EventPool(object):
         logger.info(f"Offline worker processor for queue {queue} is now active")
         
         # 扫描间隔
-        scan_interval = 1  # 每30秒扫描一次
+        scan_interval = 2  # 每30秒扫描一次
         
         while not self._stop_reading:
             try:
-                # 调用恢复模块进行恢复
-                # 不传递process_message_callback，让它使用event_queue
+                # 直接恢复所有任务（每个task都有独立的consumer group）
                 recovered = await recovery.recover_offline_workers(
                     queue=queue,
-                    event_queue=event_queue
+                    event_queue=event_queue,
+                    current_consumer_name=current_consumer
                 )
-                
                 if recovered > 0:
-                    logger.info(f"Recovered {recovered} messages for queue {queue}")
+                    logger.info(f"Recovered {recovered} messages on queue {queue}")
                     
             except Exception as e:
                 import traceback
@@ -613,9 +704,8 @@ class EventPool(object):
     
 
     async def listening_event(self, event_queue: asyncio.Queue, prefetch_multiplier: int = 1):
-        """监听事件 - 每个task都有独立的consumer group，天然支持广播"""
+        """监听事件 - 为每个task创建独立的consumer group"""
         
-        # 创建一个字典来存储每个队列的延迟任务 - 使用list + Lock更高效
         # 创建一个字典来存储每个队列的延迟任务 - 使用list + Lock更高效
         delayed_tasks_lists = {}
         delayed_tasks_locks = {}
@@ -623,6 +713,8 @@ class EventPool(object):
             delayed_tasks_lists[queue] = []
             delayed_tasks_locks[queue] = asyncio.Lock()
         
+        # group信息将在每个task监听时记录
+        
         async def scan_delayed_tasks_for_queue(queue: str, task_list: list, task_lock: asyncio.Lock):
             """为单个队列独立扫描延迟任务"""
             base_interval = self.consumer_config.get('scan_interval', 0.05)  # 基础间隔50ms
@@ -664,26 +756,98 @@ class EventPool(object):
         
         async def listen_event_by_task(queue, task_name):
             """为单个任务监听事件"""
-            check_backlog = True
-            lastid = "0-0"
+            # 为每个队列单独管理check_backlog状态
+            check_backlog = {}  # {queue_name: bool}
+            lastid = {}  # 每个队列的lastid
             consecutive_errors = 0
             max_consecutive_errors = 5
             
-            # 使用task_name作为consumer group
+            # 获取任务对象
+            task = self.app._tasks.get(task_name)
+            if not task:
+                logger.error(f"Task {task_name} not found")
+                return
+            
+            # 定义必要的变量
             prefixed_queue = self.get_prefixed_queue_name(queue)
+            # 使用函数名作为group_name，实现任务隔离
             group_name = f"{prefixed_queue}:{task_name}"
+            print(f'{group_name=} {task_name=}')
+            consumer_name = self.consumer_manager.get_consumer_name(queue)
+            
+            # 记录group信息到worker hash表
+            if self.consumer_manager:
+                await self.consumer_manager.record_group_info_async(queue, task_name, group_name, consumer_name)
+            
+            # 初始化优先级队列扫描
+            await self.update_priority_queues_cache(queue)
             
-            # 使用consumer_manager获取consumer名称，以便启动心跳等功能
-            # print(f'listening_event {queue=}')
-            managed_consumer_name = self.consumer_manager.get_consumer_name(queue)
-            # 但是为了保持每个task独立的consumer，还是需要加上task_name
-            consumer_name = f"{managed_consumer_name}:{task_name}"
+            # 获取所有优先级队列（包括默认队列）
+            priority_queues = self.get_priority_queues(queue)
+            all_queues = [prefixed_queue] + priority_queues  # 默认队列 + 优先级队列
+            
+            # 为每个队列创建consumer group（如果不存在）
+            for q in all_queues:
+                try:
+                    await self.async_redis_client.xgroup_create(
+                        name=q,
+                        groupname=group_name,
+                        id="0",
+                        mkstream=True
+                    )
+                    logger.info(f"Created consumer group {group_name} for queue {q}")
+                except Exception as e:
+                    if "BUSYGROUP" in str(e):
+                        logger.debug(f"Consumer group {group_name} already exists for queue {q}")
+                    else:
+                        logger.warning(f"Error creating consumer group for {q}: {e}")
+            
+            # 初始化每个队列的lastid和check_backlog
+            for q in all_queues:
+                lastid[q] = "0-0"
+                check_backlog[q] = True  # 每个队列都需要检查历史消息
             
             # 获取该队列的延迟任务列表和锁
             delayed_list = delayed_tasks_lists.get(queue)
             delayed_lock = delayed_tasks_locks.get(queue)
             
+            # 记录上次优先级队列更新时间
+            last_priority_update = time.time()
+            
             while not self._stop_reading:
+                # 定期更新优先级队列缓存
+                current_time = time.time()
+                if current_time - last_priority_update >= self._priority_scan_interval:
+                    await self.update_priority_queues_cache(queue)
+                    new_priority_queues = self.get_priority_queues(queue)
+                    
+                    # 如果优先级队列有变化，更新本地变量
+                    if new_priority_queues != priority_queues:
+                        logger.info(f"Priority queues updated for {queue}: {priority_queues} -> {new_priority_queues}")
+                        priority_queues = new_priority_queues
+                        all_queues = [prefixed_queue] + priority_queues
+                        
+                        # 为新的优先级队列创建consumer group
+                        for q in all_queues:
+                            if q not in lastid:  # 这是新队列
+                                try:
+                                    await self.async_redis_client.xgroup_create(
+                                        name=q,
+                                        groupname=group_name,
+                                        id="0",
+                                        mkstream=True
+                                    )
+                                    logger.info(f"Created consumer group {group_name} for new priority queue {q}")
+                                except Exception as e:
+                                    if "BUSYGROUP" not in str(e):
+                                        logger.warning(f"Error creating consumer group for {q}: {e}")
+                                
+                                # 初始化新队列的状态
+                                lastid[q] = "0-0"
+                                check_backlog[q] = True
+                    
+                    last_priority_update = current_time
+                
                 # 批量获取并处理延迟任务（使用list更高效）
                 if delayed_list:
                     # 原子地交换list内容
@@ -701,17 +865,23 @@ class EventPool(object):
                         other_tasks = []  # 属于其他task的任务
                         
                         for delayed_task in tasks_to_process:
-                            # 检查任务是否属于当前task
+                            # 对于延迟任务，不需要检查name字段
+                            # 因为延迟任务在发送时没有指定特定的task
+                            # 所有监听这个队列的task都可以处理
                             task_data = delayed_task.get('data', {})
                             if isinstance(task_data, str):
                                 import json
                                 task_data = json.loads(task_data)
                             
-                            msg_task_name = task_data.get('name')
-                            if msg_task_name == task_name:
-                                my_tasks.append((delayed_task, task_data))
-                            else:
+                            # 每个task都有独立的consumer group，都能独立处理消息
+                            # 检查消息是否指定了目标task
+                            target_tasks = task_data.get('_target_tasks', None)
+                            if target_tasks and task_name not in target_tasks:
+                                # 这个消息不是给当前task的
                                 other_tasks.append(delayed_task)
+                            else:
+                                # 当前task处理这个任务
+                                my_tasks.append((delayed_task, task_data))
                         
                         # 处理属于当前task的所有任务
                         for delayed_task, task_data in my_tasks:
@@ -726,7 +896,7 @@ class EventPool(object):
                             
                             result = self._process_message_common(
                                 event_id, task_data, queue, event_queue,
-                                is_async=True, consumer_name=consumer_name
+                                is_async=True, consumer_name=consumer_name, group_name=group_name
                             )
                             if isinstance(result, tuple) and result[0] == 'async_put':
                                 await self._async_put_task(event_queue, result[1])
@@ -736,50 +906,183 @@ class EventPool(object):
                             async with delayed_lock:
                                 delayed_list.extend(other_tasks)
                 
-                # 然后处理正常的Stream消息
-                if check_backlog:
-                    myid = lastid
-                else:
-                    myid = ">"
+                # 处理正常的Stream消息（支持优先级队列）
+                # 实现真正的优先级消费：
+                # 1. 先检查event_queue是否已满
+                # 2. 优先从高优先级队列读取
+                # 3. 只有高优先级队列空了才读取低优先级
+                # 4. 不超过prefetch_multiplier限制
                 
-                try:
-                    # 读取消息，使用二进制客户端
-                    prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-                    # group_name_bytes = group_name.encode() if isinstance(group_name, str) else group_name
-                    # consumer_name_bytes = consumer_name.encode() if isinstance(consumer_name, str) else consumer_name
+                # 检查内存队列是否已满
+                current_queue_size = event_queue.qsize() if hasattr(event_queue, 'qsize') else 0
+                if current_queue_size >= prefetch_multiplier:
+                    # 内存队列已满，等待处理
+                    await asyncio.sleep(0.01)  # 短暂等待
+                    continue
+                
+                messages = []
+                messages_needed = prefetch_multiplier - current_queue_size  # 还能读取的消息数
+                
+                if messages_needed <= 0:
+                    # 不需要读取更多消息
+                    await asyncio.sleep(0.01)
+                    continue
+                
+                # 按优先级顺序读取队列
+                for q in all_queues:
+                    if messages_needed <= 0:
+                        break  # 已经读取足够的消息
+                    
+                    q_bytes = q.encode() if isinstance(q, str) else q
+                    # 针对具体队列检查是否需要读取历史消息
+                    if check_backlog.get(q, True):
+                        myid = lastid.get(q, "0-0")
+                    else:
+                        myid = ">"
                     myid_bytes = myid.encode() if isinstance(myid, str) else myid
-                    # 动态调整阻塞时间：如果有延迟任务待处理，使用0（非阻塞）
-                    # 否则使用阻塞时间以节省CPU
-                    block_time = 1000  # 1秒阻塞
-                    # 移除debug的print语句
                     
-                    messages = await self.async_binary_redis_client.xreadgroup(
-                        groupname=group_name,
-                        consumername=consumer_name,
-                        streams={prefixed_queue_bytes: myid_bytes},
-                        count=prefetch_multiplier,
-                        block=block_time
-                    )
+                    try:
+                        # 为优先级队列使用正确的consumer名称
+                        # 如果是优先级队列，consumer名称需要带优先级后缀
+                        q_consumer_name = consumer_name
+                        if q != prefixed_queue and ':' in q:
+                            # 这是优先级队列，添加优先级后缀
+                            priority_suffix = q.rsplit(':', 1)[-1]
+                            q_consumer_name = f"{consumer_name}:{priority_suffix}"
+                        # 从当前优先级队列读取（最多读取messages_needed个）
+                        q_messages = await self.async_binary_redis_client.xreadgroup(
+                            groupname=group_name,
+                            consumername=q_consumer_name,
+                            streams={q_bytes: myid_bytes},
+                            count=messages_needed,  # 只读取需要的数量
+                            block=1  # 非阻塞
+                        )
+                        
+                        if q_messages:
+                            # logger.debug(f"Read messages from {q}: {len(q_messages[0][1]) if q_messages else 0} messages")
+                            # if check_backlog.get(q, True):
+                            #     print(f'先处理历史消息：{q_bytes=} {group_name=} {q_messages=}')
+                            # 记录从哪个队列读取的
+                            messages.extend(q_messages)
+                            messages_read = len(q_messages[0][1]) if q_messages else 0
+                            messages_needed -= messages_read
+                            
+                            # 如果高优先级队列还有消息，继续从该队列读取
+                            # 直到该队列空了或者达到prefetch限制
+                            if messages_read > 0 and messages_needed > 0:
+                                # 该队列可能还有更多消息，下次循环继续优先从这个队列读
+                                # 但现在先处理已读取的消息
+                                break  # 跳出for循环，处理已有消息
+                            
+                    except Exception as e:
+                        if "NOGROUP" in str(e):
+                            # 创建consumer group
+                            try:
+                                await self.async_redis_client.xgroup_create(
+                                    name=q,
+                                    groupname=group_name,
+                                    id="0",
+                                    mkstream=True
+                                )
+                                logger.debug(f"Created consumer group {group_name} for {q}")
+                                # 重试读取
+                                q_messages = await self.async_binary_redis_client.xreadgroup(
+                                    groupname=group_name,
+                                    consumername=q_consumer_name,
+                                    streams={q_bytes: myid_bytes},
+                                    count=messages_needed,
+                                    block=0
+                                )
+                                if q_messages:
+                                    messages.extend(q_messages)
+                                    messages_read = len(q_messages[0][1]) if q_messages else 0
+                                    messages_needed -= messages_read
+                            except:
+                                pass
+                        else:
+                            logger.debug(f"Error reading from queue {q}: {e}")
+                        continue
+                
+                # 如果没有读取到任何消息，在最高优先级队列上阻塞等待
+                if not messages:
+                    # 优先在高优先级队列上等待
+                    # 如果有优先级队列，在第一个优先级队列上等待
+                    # 否则在默认队列上等待
+                    wait_queue = all_queues[0] if all_queues else prefixed_queue
+                    
+                    try:
+                        q_bytes = wait_queue.encode() if isinstance(wait_queue, str) else wait_queue
+                        if check_backlog.get(wait_queue, True):
+                            myid = lastid.get(wait_queue, "0-0")
+                        else:
+                            myid = ">"
+                        myid_bytes = myid.encode() if isinstance(myid, str) else myid
+                        
+                        # 为等待队列使用正确的consumer名称
+                        wait_consumer_name = consumer_name
+                        if wait_queue != prefixed_queue and ':' in wait_queue:
+                            # 这是优先级队列，添加优先级后缀
+                            priority_suffix = wait_queue.rsplit(':', 1)[-1]
+                            wait_consumer_name = f"{consumer_name}:{priority_suffix}"
+                        
+                        messages = await self.async_binary_redis_client.xreadgroup(
+                            groupname=group_name,
+                            consumername=wait_consumer_name,
+                            streams={q_bytes: myid_bytes},
+                            count=prefetch_multiplier,
+                            block=100  # 阻塞100ms
+                        )
+                    except Exception as e:
+                        # 忽略错误，下次循环重试
+                        pass
+                
+                try:
                     # logger.info(f'{group_name=} {consumer_name=} {block_time=}')
                     consecutive_errors = 0
                     # if check_backlog and messages:
                     #     logger.info(f'先消费之前的消息 {group_name=} ')
                     # logger.info(f'{check_backlog=} {messages=}')
-                    if not messages:
-                        check_backlog = False
-                        # 当使用阻塞模式时，xreadgroup已经等待了block_time毫秒
-                        # 不需要额外的sleep，这会导致不必要的CPU占用
-                        continue
                     
-                    check_backlog = len(messages[0][1]) > 0
+                    # 上报已投递的offset（用于积压监控）
+                    try:
+                        from jettask.monitor.stream_backlog_monitor import report_delivered_offset
+                        # 对每个stream的消息上报offset
+                        for msg in messages:
+                            stream_name = msg[0]
+                            if isinstance(stream_name, bytes):
+                                stream_name = stream_name.decode('utf-8')
+                            # 提取队列名（去掉前缀）
+                            queue_name = stream_name.replace(f"{self.redis_prefix}:STREAM:", "")
+                            await report_delivered_offset(
+                                self.async_redis_client,
+                                self.redis_prefix,
+                                queue_name,
+                                group_name,
+                                [msg]
+                            )
+                    except Exception as e:
+                        # 监控失败不影响主流程
+                        logger.debug(f"Failed to report delivered offset: {e}")
                     
                     # 收集需要跳过的消息ID
                     skip_message_ids = []
                     
                     for message in messages:
+                        # print(f'{message=}')
+                        # message[0]是stream名称，message[1]是消息列表
+                        stream_name = message[0]
+                        if isinstance(stream_name, bytes):
+                            stream_name = stream_name.decode('utf-8')
+                        
+                        # 根据这个具体队列的消息数量，更新该队列的check_backlog状态
+                        if len(message[1]) == 0:
+                            # 这个队列没有历史消息了，下次读取最新消息
+                            check_backlog[stream_name] = False
+                        
                         for event in message[1]:
                             event_id = event[0]
-                            lastid = event_id
+                            # 更新对应队列的lastid
+                            lastid[stream_name] = event_id
                             # 将bytes类型的event_id转换为字符串
                             if isinstance(event_id, bytes):
                                 event_id = event_id.decode('utf-8')
@@ -795,30 +1098,30 @@ class EventPool(object):
                                     
                                     # 直接解析二进制数据，不需要解码
                                     parsed_data = loads_str(data_field)
-                                    
                                     # 跳过延迟任务（延迟任务由延迟扫描器处理）
                                     if parsed_data.get('is_delayed') == 1:
                                         should_process = False
                                         continue
                                     
-                                    # 检查是否是广播消息
-                                    is_broadcast = parsed_data.get('_broadcast', False)
+                                    # 每个task都有独立的consumer group
+                                    # 检查消息是否指定了目标task（用于精确路由）
                                     target_tasks = parsed_data.get('_target_tasks', None)
-                                    
-                                    if is_broadcast:
-                                        # 广播消息：默认所有task都处理，除非被target_tasks排除
-                                        if target_tasks and task_name not in target_tasks:
-                                            should_process = False
-                                    else:
-                                        # 普通消息：必须有name字段且匹配当前task才处理
-                                        msg_task_name = parsed_data.get('name')
-                                        if not msg_task_name or msg_task_name != task_name:
-                                            should_process = False
+                                    if target_tasks and task_name not in target_tasks:
+                                        # 这个消息指定了其他task处理
+                                        should_process = False
                                     
                                     if should_process:
                                         # 添加task_name到数据中（用于执行器识别任务）
                                         parsed_data['_task_name'] = task_name
                                         
+                                        # 提取offset字段（如果存在）
+                                        offset_field = event_data.get(b'offset') or event_data.get('offset')
+                                        if offset_field:
+                                            # 将offset添加到parsed_data中
+                                            if isinstance(offset_field, bytes):
+                                                offset_field = offset_field.decode('utf-8')
+                                            parsed_data['offset'] = offset_field
+                                        
                                         # 更新event_data
                                         event_data.clear()
                                         for key, value in parsed_data.items():
@@ -833,22 +1136,40 @@ class EventPool(object):
                             
                             if should_process:
                                 # 处理消息 - 消息会被放入队列，由执行器处理并ACK
+                                # 使用消息体中的实际队列名（可能包含优先级）
+                                actual_queue = event_data.get('queue', queue)
+                                
+                                # 确定实际的consumer名称（对于优先级队列需要带后缀）
+                                actual_consumer_name = consumer_name
+                                # 从stream_name判断是否是优先级队列
+                                if stream_name != prefixed_queue and ':' in stream_name:
+                                    # 这是优先级队列，添加优先级后缀
+                                    priority_suffix = stream_name.rsplit(':', 1)[-1]
+                                    actual_consumer_name = f"{consumer_name}:{priority_suffix}"
+                                
                                 result = self._process_message_common(
-                                    event_id, event_data, queue, event_queue,
-                                    is_async=True, consumer_name=consumer_name
+                                    event_id, event_data, actual_queue, event_queue,
+                                    is_async=True, consumer_name=actual_consumer_name, group_name=group_name
                                 )
                                 if isinstance(result, tuple) and result[0] == 'async_put':
                                     await self._async_put_task(event_queue, result[1])
+                                    logger.debug(f"Put task {event_id} into event_queue")
                                 # 注意：这里不ACK，由执行器在处理完成后ACK
                             else:
                                 # 不属于当前task的消息，收集起来批量ACK
                                 skip_message_ids.append(event_id)
                             
                     
-                    # 批量ACK不需要的消息
+                    # 批量ACK不需要的消息（需要按队列分组）
                     if skip_message_ids:
+                        # 这里简化处理，对所有队列都尝试ACK（实际只会在对应队列ACK成功）
                         group_name_bytes = group_name.encode() if isinstance(group_name, str) else group_name
-                        await self.async_binary_redis_client.xack(prefixed_queue_bytes, group_name_bytes, *skip_message_ids)
+                        for q in all_queues:
+                            q_bytes = q.encode() if isinstance(q, str) else q
+                            try:
+                                await self.async_binary_redis_client.xack(q_bytes, group_name_bytes, *skip_message_ids)
+                            except:
+                                pass  # 忽略ACK错误
                         logger.debug(f"Task {task_name} batch ACKed {len(skip_message_ids)} skipped messages")
                         
                 except Exception as e:
@@ -861,13 +1182,19 @@ class EventPool(object):
                     if "NOGROUP" in error_msg:
                         logger.info(f"Detected NOGROUP error for {task_name}, attempting to recreate consumer group...")
                         try:
-                            await self.async_redis_client.xgroup_create(
-                                name=prefixed_queue,
-                                groupname=group_name,
-                                id="0",
-                                mkstream=True
-                            )
-                            logger.info(f"Successfully recreated consumer group {group_name} for task {task_name}")
+                            # 为所有队列创建consumer group
+                            for q in all_queues:
+                                try:
+                                    await self.async_redis_client.xgroup_create(
+                                        name=q,
+                                        groupname=group_name,
+                                        id="0",
+                                        mkstream=True
+                                    )
+                                    logger.info(f"Successfully created consumer group {group_name} for queue {q}")
+                                except:
+                                    pass  # 可能已存在
+                            logger.info(f"Consumer groups created/verified for task {task_name}")
                             # 重新创建成功，重置错误计数器
                             consecutive_errors = 0
                             continue
@@ -899,7 +1226,17 @@ class EventPool(object):
             tasks.append(scanner_task)
         
         # 为每个队列启动离线worker处理器（带自动重启）
-        for queue in self.queues:
+        # 包括优先级队列
+        all_recovery_queues = set(self.queues)
+        for base_queue in self.queues:
+            # 扫描优先级队列
+            priority_queues = await self.scan_priority_queues(base_queue)
+            for pq in priority_queues:
+                if pq != base_queue:  # 不重复添加基础队列
+                    all_recovery_queues.add(pq)
+        
+        # 为所有队列（包括优先级队列）启动离线worker处理器
+        for queue in all_recovery_queues:
             logger.info(f"Starting offline worker processor for queue: {queue}")
             offline_processor_task = asyncio.create_task(
                 self._start_offline_worker_processor_with_restart(queue, event_queue)
@@ -1008,7 +1345,7 @@ class EventPool(object):
             if tasks_to_add:
                 async with task_lock:
                     task_list.extend(tasks_to_add)
-                logger.debug(f"Added {len(tasks_to_add)} delayed tasks to list for queue {queue}")
+                logger.info(f"Added {len(tasks_to_add)} delayed tasks to list for queue {queue}")
                     
         except Exception as e:
             logger.error(f"Error scanning delayed tasks for queue {queue}: {e}")
@@ -1241,20 +1578,16 @@ class EventPool(object):
             logger.error(f"Error processing delayed tasks for queue {queue}: {e}")
             # 错误不应该阻塞主流程
     def read_pending(self, groupname: str, queue: str, asyncio: bool = False):
-        # 使用二进制客户端进行Stream操作
+        # 现在使用自动解码的客户端
         client = self.get_redis_client(asyncio, binary=True)
         prefixed_queue = self.get_prefixed_queue_name(queue)
-        prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-        groupname_bytes = groupname.encode() if isinstance(groupname, str) else groupname
-        return client.xpending(prefixed_queue_bytes, groupname_bytes)
+        return client.xpending(prefixed_queue, groupname)
 
     def ack(self, queue, event_id, asyncio: bool = False):
-        # 使用二进制客户端进行Stream操作
+        # 现在使用自动解码的客户端
         client = self.get_redis_client(asyncio, binary=True)
         prefixed_queue = self.get_prefixed_queue_name(queue)
-        prefixed_queue_bytes = prefixed_queue.encode() if isinstance(prefixed_queue, str) else prefixed_queue
-        event_id_bytes = event_id if isinstance(event_id, bytes) else event_id.encode() if isinstance(event_id, str) else str(event_id).encode()
-        result = client.xack(prefixed_queue_bytes, prefixed_queue_bytes, event_id_bytes)
+        result = client.xack(prefixed_queue, prefixed_queue, event_id)
         # 清理已认领的消息ID
         if event_id in self._claimed_message_ids:
             self._claimed_message_ids.remove(event_id)