jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/db/models/task.py

@@ -0,0 +1,77 @@
+"""
+Task 模型
+
+对应 tasks 表，用于存储任务消息（实际表结构）
+"""
+from sqlalchemy import Column, String, Integer, Text, TIMESTAMP, Index
+from sqlalchemy.dialects.postgresql import JSONB
+from datetime import datetime
+from typing import Optional, Dict, Any
+
+from ..base import Base
+
+
+class Task(Base):
+    """
+    任务消息表
+
+    存储发送到队列的任务消息
+    """
+    __tablename__ = 'tasks'
+
+    # 主键 - Redis Stream ID
+    stream_id = Column(Text, primary_key=True, comment='Redis Stream 事件ID')
+
+    # 队列和命名空间
+    queue = Column(Text, nullable=False, comment='队列名称')
+    namespace = Column(Text, nullable=False, comment='命名空间')
+
+    # 定时任务关联
+    scheduled_task_id = Column(Text, nullable=True, comment='关联的定时任务ID')
+
+    # 消息数据
+    payload = Column(JSONB, nullable=False, comment='消息载荷（任务数据）')
+
+    # 优先级
+    priority = Column(Integer, nullable=True, comment='优先级（数字越小优先级越高）')
+
+    # 时间戳
+    created_at = Column(
+        TIMESTAMP(timezone=True),
+        nullable=False,
+        default=datetime.utcnow,
+        comment='创建时间'
+    )
+
+    # 来源
+    source = Column(Text, nullable=False, comment='消息来源（如 scheduler, manual, api）')
+
+    # 元数据
+    task_metadata = Column('metadata', JSONB, nullable=True, comment='任务元数据')
+
+    # 索引
+    __table_args__ = (
+        Index('idx_tasks_queue', 'queue'),
+        Index('idx_tasks_namespace', 'namespace'),
+        Index('idx_tasks_queue_namespace', 'queue', 'namespace'),
+        Index('idx_tasks_scheduled_task_id', 'scheduled_task_id'),
+        Index('idx_tasks_created_at', 'created_at'),
+        Index('idx_tasks_source', 'source'),
+    )
+
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return {
+            'stream_id': self.stream_id,
+            'queue': self.queue,
+            'namespace': self.namespace,
+            'scheduled_task_id': self.scheduled_task_id,
+            'payload': self.payload,
+            'priority': self.priority,
+            'created_at': self.created_at.isoformat() if self.created_at else None,
+            'source': self.source,
+            'metadata': self.task_metadata,
+        }
+
+    def __repr__(self) -> str:
+        return f"<Task(stream_id='{self.stream_id}', queue='{self.queue}', namespace='{self.namespace}')>"

jettask/db/models/task_run.py

@@ -0,0 +1,85 @@
+"""
+TaskRun 模型
+
+对应 task_runs 表，用于存储任务执行记录
+"""
+from sqlalchemy import Column, String, Integer, Text, TIMESTAMP, Float, Index, ForeignKey
+from sqlalchemy.dialects.postgresql import JSONB
+from datetime import datetime
+from typing import Optional, Dict, Any
+
+from ..base import Base
+
+
+class TaskRun(Base):
+    """
+    任务执行记录表
+
+    存储每次任务执行的状态、结果和执行时间等信息
+    """
+    __tablename__ = 'task_runs'
+
+    # 主键 - Redis Stream ID
+    stream_id = Column(Text, primary_key=True, comment='Redis Stream 事件ID，关联到 tasks 表')
+
+    # 执行状态
+    status = Column(String(50), nullable=True, comment='任务状态 (pending/running/success/failed/retrying)')
+
+    # 执行结果
+    result = Column(JSONB, nullable=True, comment='任务执行结果')
+    error = Column(Text, nullable=True, comment='错误信息（如果失败）')
+
+    # 执行时间
+    started_at = Column(Float, nullable=True, comment='开始执行时间（Unix 时间戳）')
+    completed_at = Column(Float, nullable=True, comment='完成时间（Unix 时间戳）')
+
+    # 重试次数
+    retries = Column(Integer, nullable=True, default=0, comment='重试次数')
+
+    # 执行时长（秒）
+    duration = Column(Float, nullable=True, comment='执行时长（秒）')
+
+    # 消费者信息
+    consumer = Column(Text, nullable=True, comment='执行该任务的消费者ID')
+
+    # 记录创建和更新时间
+    created_at = Column(
+        TIMESTAMP(timezone=True),
+        nullable=False,
+        default=datetime.utcnow,
+        comment='记录创建时间'
+    )
+    updated_at = Column(
+        TIMESTAMP(timezone=True),
+        nullable=False,
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow,
+        comment='记录更新时间'
+    )
+
+    # 索引
+    __table_args__ = (
+        Index('idx_task_runs_status', 'status'),
+        Index('idx_task_runs_started_at', 'started_at'),
+        Index('idx_task_runs_completed_at', 'completed_at'),
+        Index('idx_task_runs_created_at', 'created_at'),
+    )
+
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return {
+            'stream_id': self.stream_id,
+            'status': self.status,
+            'result': self.result,
+            'error': self.error,
+            'started_at': self.started_at,
+            'completed_at': self.completed_at,
+            'retries': self.retries,
+            'duration': self.duration,
+            'consumer': self.consumer,
+            'created_at': self.created_at.isoformat() if self.created_at else None,
+            'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+        }
+
+    def __repr__(self) -> str:
+        return f"<TaskRun(stream_id='{self.stream_id}', status='{self.status}')>"

jettask/executor/__init__.py

@@ -13,18 +13,6 @@ from .core import ExecutorCore
 from .orchestrator import ProcessConfig, ProcessOrchestrator
 from .task_executor import TaskExecutor
 
-# 保留 UnifiedExecutor 作为兼容（已废弃）
-try:
-    from .executor import UnifiedExecutor
-except ImportError:
-    # 如果导入失败，提供一个废弃提示
-    class UnifiedExecutor:
-        def __init__(self, *args, **kwargs):
-            raise DeprecationWarning(
-                "UnifiedExecutor is deprecated. "
-                "Use TaskExecutor for single task execution, "
-                "or ProcessOrchestrator for multi-process management."
-            )
 
 __all__ = [
     # 新的推荐类
@@ -32,7 +20,4 @@ __all__ = [
     'ProcessOrchestrator',
     'ProcessConfig',
     'ExecutorCore',
-
-    # 废弃但保留兼容性
-    'UnifiedExecutor',
 ]

jettask/executor/core.py

@@ -14,7 +14,7 @@ import logging
 import time
 import os
 from enum import Enum
-from typing import Dict, Optional
+from typing import Dict, Optional, TYPE_CHECKING
 from collections import defaultdict, deque
 
 from ..utils.traceback_filter import filter_framework_traceback
@@ -23,6 +23,9 @@ from ..utils.serializer import dumps_str
 from ..exceptions import RetryableError
 from ..core.enums import TaskStatus
 from ..utils.rate_limit.limiter import RateLimiterManager, ConcurrencyRateLimiter
+from ..config.lua_scripts import LUA_SCRIPT_BATCH_SEND_EVENT
+if TYPE_CHECKING:
+    from ..core.app import Jettask
 
 logger = logging.getLogger('app')
 
@@ -61,7 +64,7 @@ class ExecutorCore:
     4. 统计收集
     """
 
-    def __init__(self, app, task_name: str, concurrency: int = 100):
+    def __init__(self, app: "Jettask", task_name: str, concurrency: int = 100):
         """
         初始化执行器核心
 
@@ -218,39 +221,37 @@ class ExecutorCore:
                     operations_count += 1
 
             self.pending_acks.clear()
-
         # 2. 处理任务信息更新
         task_change_events = []
         if self.task_info_updates:
             for event_key, updates in self.task_info_updates.items():
-                key = f"{self.prefix}:TASK:{event_key}".encode()
+                if event_key.endswith('consumer._handle_status_update') or \
+                    event_key.endswith('consumer._handle_persist_task'):
+                    continue  # 跳过无效的event_key
+                key = f"{self.prefix}:TASK:{event_key}"
+                key_bytes = key.encode() 
                 if updates:
                     encoded_updates = {k.encode(): v.encode() if isinstance(v, str) else v
                                      for k, v in updates.items()}
-                    pipeline.hset(key, mapping=encoded_updates)
-                    pipeline.expire(key, 3600)
+                    pipeline.hset(key_bytes, mapping=encoded_updates)
+                    pipeline.expire(key_bytes, 3600)
                     operations_count += 2
+                    task_change_events.append(key)
 
-                    full_task_id = f"{self.prefix}:TASK:{event_key}"
-                    task_change_events.append(full_task_id)
-
-            # 发送变更事件
-            change_stream_key = f"{self.prefix}:TASK_CHANGES".encode()
-            for task_id in task_change_events:
-                change_data = {b'id': task_id.encode() if isinstance(task_id, str) else task_id}
-                pipeline.xadd(change_stream_key, change_data, maxlen=1000000)
-                operations_count += 1
+            # 发送变更事件（使用 Lua 脚本确保 offset 更新和 REGISTRY 注册）
+            if task_change_events:
+                self._send_task_changes_with_offset(task_change_events, pipeline)
 
             self.task_info_updates.clear()
 
         # 3. 处理统计信息
         if hasattr(self, 'stats_updates') and self.stats_updates:
-            for stat_op in self.stats_updates:
-                if 'queue' in stat_op and 'field' in stat_op:
-                    stats_key = f"{self.prefix}:STATS:{stat_op['queue']}".encode()
-                    field = stat_op['field'].encode() if isinstance(stat_op['field'], str) else stat_op['field']
-                    pipeline.hincrby(stats_key, field, stat_op.get('value', 1))
-                    operations_count += 1
+            # for stat_op in self.stats_updates:
+            #     if 'queue' in stat_op and 'field' in stat_op:
+            #         stats_key = f"{self.prefix}:STATS:{stat_op['queue']}".encode()
+            #         field = stat_op['field'].encode() if isinstance(stat_op['field'], str) else stat_op['field']
+            #         pipeline.hincrby(stats_key, field, stat_op.get('value', 1))
+            #         operations_count += 1
             self.stats_updates.clear()
 
         # 执行pipeline
@@ -427,10 +428,20 @@ class ExecutorCore:
                         if current_retry > 0:
                             logger.debug(f"Retry attempt {current_retry}/{max_retries} for task {event_id}")
 
+                        # 不过滤 __ 开头的参数，因为它们需要在 _inject_dependencies 中被提取
+                        # （如 __scheduled_task_id, __queue 等）
+                        # 只过滤单下划线开头的参数
                         clean_kwargs = {k: v for k, v in kwargs_inner.items()
-                                      if not k.startswith('_') and not k.startswith('__')}
-
-                        task_result = task(event_id, event_data['trigger_time'], *args, **clean_kwargs)
+                                      if not k.startswith('_') or k.startswith('__')}
+
+                        print(f'{queue=} {args=} {kwargs_inner=}')
+                        task_result = task(
+                            event_id, 
+                            event_data.get('trigger_time'), 
+                            queue,
+                            *args, 
+                            **clean_kwargs
+                        )
                         if asyncio.iscoroutine(task_result):
                             ret = await task_result
                         else:
@@ -445,13 +456,14 @@ class ExecutorCore:
                         if asyncio.iscoroutine(result):
                             await result
 
-                        # 任务成功,ACK消息
-                        offset = self._extract_offset(event_data)
-                        # await self.app.ep.async_redis_client.xack(
-                        #     self._get_prefixed_queue_cached(queue), group_name, event_id
-                        # )
-                        # await self.app.ep.async_redis_client.close()
-                        await self._quick_ack(queue, event_id, group_name, offset)
+                        # 任务成功,ACK消息（检查auto_ack配置）
+                        task_config = self.app.get_task_config(task_name)
+                        auto_ack = task_config.get('auto_ack', True) if task_config else True
+
+                        if auto_ack:
+                            offset = self._extract_offset(event_data)
+                            await self._quick_ack(queue, event_id, group_name, offset)
+
                         break
 
                     except SystemExit:
@@ -599,6 +611,39 @@ class ExecutorCore:
                     pass
         return None
 
+    def _send_task_changes_with_offset(self, task_ids: list, pipeline):
+        """发送 TASK_CHANGES 消息并自动更新 offset
+
+        使用 Lua 脚本确保：
+        1. 自动递增 offset
+        2. 注册队列到 REGISTRY:QUEUES
+        3. 消息中包含 offset 字段
+
+        Args:
+            task_ids: 任务 ID 列表
+            pipeline: Redis pipeline
+        """
+        from jettask.utils.serializer import dumps_str
+
+        # 准备Lua脚本参数
+        stream_key = f"{self.prefix}:QUEUE:TASK_CHANGES".encode()
+        lua_args = [self.prefix.encode() if isinstance(self.prefix, str) else self.prefix]
+
+        # 为每个 task_id 构建消息
+        for task_id in task_ids:
+            message_data = {'kwargs': {'task_id': task_id}}
+            data = dumps_str(message_data)
+            lua_args.append(data)
+
+        # 使用 pipeline 执行 Lua 脚本
+        pipeline.eval(
+            LUA_SCRIPT_BATCH_SEND_EVENT,
+            1,  # 1个KEY
+            stream_key,  # KEY[1]: stream key
+            *lua_args  # ARGV: prefix, data1, data2, ...
+        )
+        logger.debug(f"已添加 {len(task_ids)} 条 TASK_CHANGES 消息到 pipeline")
+
     async def cleanup(self):
         """清理资源"""
         logger.debug("ExecutorCore cleaning up...")

jettask/executor/process_entry.py

@@ -16,116 +16,11 @@ import multiprocessing
 from typing import List, Dict
 import time 
 
-
-
-
-    
-import redis 
-async def execute_single_command(client: redis.Redis, index: int):
-    """
-    执行单条 Redis 命令
-
-    Args:
-        client: Redis 客户端
-        index: 命令索引
-    """
-    await client.set(f"test_key_{index}", f"value_{index}")
-
-
-async def execute_commands(client: redis.Redis, num_commands: int = 10000, concurrency: int = 100):
-    """
-    并发执行大量 Redis 命令
-
-    Args:
-        client: Redis 客户端
-        num_commands: 命令数量
-        concurrency: 并发数（同时执行的任务数）
-    """
-    logger.info(f"开始并发执行 {num_commands} 条 Redis 命令（并发数: {concurrency}）...")
-
-    start_time = time.time()
-
-    # 创建信号量控制并发数
-    semaphore = asyncio.Semaphore(concurrency)
-
-    async def execute_with_semaphore(index: int):
-        """使用信号量控制并发"""
-        # async with semaphore:
-        await execute_single_command(client, index)
-
-        # 每 1000 条命令打印一次进度
-        if (index + 1) % 1000 == 0:
-            logger.info(f"已完成 {index + 1} 条命令")
-
-    # 创建所有任务
-    tasks = [
-        asyncio.create_task(execute_with_semaphore(i))
-        for i in range(num_commands)
-    ]
-
-    # 等待所有任务完成
-    await asyncio.gather(*tasks)
-
-    elapsed = time.time() - start_time
-    logger.info(f"✅ 完成 {num_commands} 条命令，耗时: {elapsed:.2f}s，QPS: {num_commands/elapsed:.0f}")
-    await asyncio.sleep(999999999999999999)
-    return elapsed
-
-
 # 不要在模块级别创建 logger，避免在 fork 时触发 logging 全局锁竞争
 # logger 将在 subprocess_main 中创建
 logger = None
 
 
-def _reset_locks_after_fork():
-    """
-    在 fork 后立即重置所有 logging 相关的锁
-
-    关键：不能调用任何 logging API（如 getLogger），因为这些 API 本身需要获取锁。
-    必须直接访问 logging 模块的内部数据结构。
-    """
-    import threading
-    import logging
-
-    # 1. 暴力替换 logging 模块的全局锁（不通过 API）
-    logging._lock = threading.RLock()
-
-    # 2. 直接访问 Logger.manager 的内部字典，不调用 getLogger()
-    # 这样避免触发锁的获取
-    if hasattr(logging.Logger, 'manager'):
-        manager = logging.Logger.manager
-
-        # 2.1 重置 manager 的锁
-        if hasattr(manager, 'lock'):
-            manager.lock = threading.RLock()
-
-        # 2.2 重置根 logger 的锁（直接访问 root 属性）
-        if hasattr(manager, 'root'):
-            root = manager.root
-            if hasattr(root, '_lock'):
-                root._lock = threading.RLock()
-
-            # 重置根 logger 的所有 handlers 的锁
-            if hasattr(root, 'handlers'):
-                for handler in list(root.handlers):
-                    if hasattr(handler, 'lock'):
-                        handler.lock = threading.RLock()
-
-        # 2.3 重置所有子 logger 的锁（直接访问字典，不调用 getLogger）
-        if hasattr(manager, 'loggerDict'):
-            for logger_obj in list(manager.loggerDict.values()):
-                # loggerDict 中的值可能是 PlaceHolder 或 Logger
-                if hasattr(logger_obj, '_lock'):
-                    logger_obj._lock = threading.RLock()
-
-                # 重置 handlers 的锁
-                if hasattr(logger_obj, 'handlers'):
-                    for handler in list(logger_obj.handlers):
-                        if hasattr(handler, 'lock'):
-                            handler.lock = threading.RLock()
-
-
-
 class SubprocessInitializer:
     """子进程初始化器 - 负责清理和准备环境"""
 
@@ -160,7 +55,7 @@ class SubprocessInitializer:
 
         # 3. 清空Redis连接池和客户端缓存
         # 这非常重要！防止子进程复用父进程的连接
-        from jettask.utils.db_connector import clear_all_cache
+        from jettask.db.connector import clear_all_cache
         clear_all_cache()
 
         # 4. 强制垃圾回收
@@ -291,6 +186,28 @@ class MinimalApp:
         """根据任务名称获取任务对象"""
         return self._tasks.get(task_name)
 
+    def get_task_config(self, task_name: str) -> dict:
+        """
+        获取任务配置
+
+        Args:
+            task_name: 任务名称
+
+        Returns:
+            任务配置字典，如果任务不存在则返回None
+        """
+        task = self.get_task_by_name(task_name)
+        if not task:
+            return None
+
+        return {
+            'auto_ack': getattr(task, 'auto_ack', True),
+            'queue': getattr(task, 'queue', None),
+            'timeout': getattr(task, 'timeout', None),
+            'max_retries': getattr(task, 'max_retries', 0),
+            'retry_delay': getattr(task, 'retry_delay', None),
+        }
+
     def cleanup(self):
         """清理资源"""
         pass
@@ -345,12 +262,12 @@ class SubprocessRunner:
             if self.event_pool:
                 self.event_pool._stop_reading = True
 
-        # signal.signal(signal.SIGINT, signal_handler)
-        # signal.signal(signal.SIGTERM, signal_handler)
+        signal.signal(signal.SIGINT, signal_handler)
+        signal.signal(signal.SIGTERM, signal_handler)
 
     def create_redis_connections(self):
         """创建独立的Redis连接（使用全局客户端实例）"""
-        from jettask.utils.db_connector import get_sync_redis_client, get_async_redis_client
+        from jettask.db.connector import get_sync_redis_client, get_async_redis_client
 
         # logger.info(f"Process #{self.process_id}: Creating Redis connections")
 
@@ -394,16 +311,13 @@ class SubprocessRunner:
         self.event_pool = EventPool(
             self.redis_client,
             self.async_redis_client,
+            queues=self.queues,  # ✅ 传递 queues 参数以支持通配符模式
             redis_url=self.redis_url,
             consumer_strategy=self.consumer_strategy,
             consumer_config=consumer_config,
             redis_prefix=self.redis_prefix,
             app=self.minimal_app
         )
-        
-  
-        # logger.info('准备进入测试流程')
-        # await execute_commands(self.event_pool.async_redis_client, num_commands=100000)
 
         # 将 EventPool 设置到 MinimalApp
         self.minimal_app.ep = self.event_pool
@@ -418,7 +332,8 @@ class SubprocessRunner:
         )
 
         # 初始化路由
-        self.event_pool.queues = self.queues
+        # ✅ 不再需要这行，因为 EventPool.__init__ 已经正确处理了 queues（包括通配符）
+        # self.event_pool.queues = self.queues
         self.event_pool.init_routing()
 
         # 收集任务（按队列分组）

jettask/executor/task_executor.py

@@ -56,6 +56,10 @@ class TaskExecutor:
             concurrency=concurrency
         )
 
+        # 将executor_core存储到app，供手动ACK使用
+        if not hasattr(app, '_executor_core'):
+            app._executor_core = self.executor_core
+
         # 活动任务集合
         self._active_tasks: Set[asyncio.Task] = set()