jettask 0.2.20__py3-none-any.whl → 0.2.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/__init__.py +4 -0
- jettask/cli.py +12 -8
- jettask/config/lua_scripts.py +37 -0
- jettask/config/nacos_config.py +1 -1
- jettask/core/app.py +313 -340
- jettask/core/container.py +4 -4
- jettask/{persistence → core}/namespace.py +93 -27
- jettask/core/task.py +16 -9
- jettask/core/unified_manager_base.py +136 -26
- jettask/db/__init__.py +67 -0
- jettask/db/base.py +137 -0
- jettask/{utils/db_connector.py → db/connector.py} +130 -26
- jettask/db/models/__init__.py +16 -0
- jettask/db/models/scheduled_task.py +196 -0
- jettask/db/models/task.py +77 -0
- jettask/db/models/task_run.py +85 -0
- jettask/executor/__init__.py +0 -15
- jettask/executor/core.py +76 -31
- jettask/executor/process_entry.py +29 -114
- jettask/executor/task_executor.py +4 -0
- jettask/messaging/event_pool.py +928 -685
- jettask/messaging/scanner.py +30 -0
- jettask/persistence/__init__.py +28 -103
- jettask/persistence/buffer.py +170 -0
- jettask/persistence/consumer.py +330 -249
- jettask/persistence/manager.py +304 -0
- jettask/persistence/persistence.py +391 -0
- jettask/scheduler/__init__.py +15 -3
- jettask/scheduler/{task_crud.py → database.py} +61 -57
- jettask/scheduler/loader.py +2 -2
- jettask/scheduler/{scheduler_coordinator.py → manager.py} +23 -6
- jettask/scheduler/models.py +14 -10
- jettask/scheduler/schedule.py +166 -0
- jettask/scheduler/scheduler.py +12 -11
- jettask/schemas/__init__.py +50 -1
- jettask/schemas/backlog.py +43 -6
- jettask/schemas/namespace.py +70 -19
- jettask/schemas/queue.py +19 -3
- jettask/schemas/responses.py +493 -0
- jettask/task/__init__.py +0 -2
- jettask/task/router.py +3 -0
- jettask/test_connection_monitor.py +1 -1
- jettask/utils/__init__.py +7 -5
- jettask/utils/db_init.py +8 -4
- jettask/utils/namespace_dep.py +167 -0
- jettask/utils/queue_matcher.py +186 -0
- jettask/utils/rate_limit/concurrency_limiter.py +7 -1
- jettask/utils/stream_backlog.py +1 -1
- jettask/webui/__init__.py +0 -1
- jettask/webui/api/__init__.py +4 -4
- jettask/webui/api/alerts.py +806 -71
- jettask/webui/api/example_refactored.py +400 -0
- jettask/webui/api/namespaces.py +390 -45
- jettask/webui/api/overview.py +300 -54
- jettask/webui/api/queues.py +971 -267
- jettask/webui/api/scheduled.py +1249 -56
- jettask/webui/api/settings.py +129 -7
- jettask/webui/api/workers.py +442 -0
- jettask/webui/app.py +46 -2329
- jettask/webui/middleware/__init__.py +6 -0
- jettask/webui/middleware/namespace_middleware.py +135 -0
- jettask/webui/services/__init__.py +146 -0
- jettask/webui/services/heartbeat_service.py +251 -0
- jettask/webui/services/overview_service.py +60 -51
- jettask/webui/services/queue_monitor_service.py +426 -0
- jettask/webui/services/redis_monitor_service.py +87 -0
- jettask/webui/services/settings_service.py +174 -111
- jettask/webui/services/task_monitor_service.py +222 -0
- jettask/webui/services/timeline_pg_service.py +452 -0
- jettask/webui/services/timeline_service.py +189 -0
- jettask/webui/services/worker_monitor_service.py +467 -0
- jettask/webui/utils/__init__.py +11 -0
- jettask/webui/utils/time_utils.py +122 -0
- jettask/worker/lifecycle.py +8 -2
- {jettask-0.2.20.dist-info → jettask-0.2.24.dist-info}/METADATA +1 -1
- jettask-0.2.24.dist-info/RECORD +142 -0
- jettask/executor/executor.py +0 -338
- jettask/persistence/backlog_monitor.py +0 -567
- jettask/persistence/base.py +0 -2334
- jettask/persistence/db_manager.py +0 -516
- jettask/persistence/maintenance.py +0 -81
- jettask/persistence/message_consumer.py +0 -259
- jettask/persistence/models.py +0 -49
- jettask/persistence/offline_recovery.py +0 -196
- jettask/persistence/queue_discovery.py +0 -215
- jettask/persistence/task_persistence.py +0 -218
- jettask/persistence/task_updater.py +0 -583
- jettask/scheduler/add_execution_count.sql +0 -11
- jettask/scheduler/add_priority_field.sql +0 -26
- jettask/scheduler/add_scheduler_id.sql +0 -25
- jettask/scheduler/add_scheduler_id_index.sql +0 -10
- jettask/scheduler/make_scheduler_id_required.sql +0 -28
- jettask/scheduler/migrate_interval_seconds.sql +0 -9
- jettask/scheduler/performance_optimization.sql +0 -45
- jettask/scheduler/run_scheduler.py +0 -186
- jettask/scheduler/schema.sql +0 -84
- jettask/task/task_executor.py +0 -318
- jettask/webui/api/analytics.py +0 -323
- jettask/webui/config.py +0 -90
- jettask/webui/models/__init__.py +0 -3
- jettask/webui/models/namespace.py +0 -63
- jettask/webui/namespace_manager/__init__.py +0 -10
- jettask/webui/namespace_manager/multi.py +0 -593
- jettask/webui/namespace_manager/unified.py +0 -193
- jettask/webui/run.py +0 -46
- jettask-0.2.20.dist-info/RECORD +0 -145
- {jettask-0.2.20.dist-info → jettask-0.2.24.dist-info}/WHEEL +0 -0
- {jettask-0.2.20.dist-info → jettask-0.2.24.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.20.dist-info → jettask-0.2.24.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.20.dist-info → jettask-0.2.24.dist-info}/top_level.txt +0 -0
jettask/core/app.py
CHANGED
@@ -26,7 +26,7 @@ from ..executor.orchestrator import ProcessOrchestrator
|
|
26
26
|
from ..utils import gen_task_name
|
27
27
|
from ..exceptions import TaskTimeoutError, TaskExecutionError, TaskNotFoundError
|
28
28
|
# 导入统一的数据库连接管理
|
29
|
-
from
|
29
|
+
from jettask.db.connector import get_sync_redis_client, get_async_redis_client
|
30
30
|
# 导入Lua脚本
|
31
31
|
from ..config.lua_scripts import (
|
32
32
|
LUA_SCRIPT_DELAYED_TASKS,
|
@@ -351,12 +351,15 @@ class Jettask(object):
|
|
351
351
|
if self.task_center and self.task_center.is_enabled and not self._task_center_config:
|
352
352
|
self._load_config_from_task_center()
|
353
353
|
|
354
|
-
#
|
354
|
+
# 使用无限超时,支持 Pub/Sub 长连接(可能几天没有消息)
|
355
|
+
import logging
|
356
|
+
logger = logging.getLogger(__name__)
|
357
|
+
logger.info(f"Creating async_redis client with socket_timeout=None for redis_url={self.redis_url}")
|
355
358
|
return get_async_redis_client(
|
356
359
|
redis_url=self.redis_url,
|
357
360
|
decode_responses=True,
|
358
361
|
max_connections=self.max_connections,
|
359
|
-
socket_timeout=
|
362
|
+
socket_timeout=None # 无限等待,不超时
|
360
363
|
)
|
361
364
|
|
362
365
|
@property
|
@@ -388,7 +391,7 @@ class Jettask(object):
|
|
388
391
|
redis_url=self.redis_url,
|
389
392
|
decode_responses=False,
|
390
393
|
max_connections=self.max_connections,
|
391
|
-
socket_timeout=
|
394
|
+
socket_timeout=None
|
392
395
|
)
|
393
396
|
|
394
397
|
@property
|
@@ -456,11 +459,35 @@ class Jettask(object):
|
|
456
459
|
|
457
460
|
return None
|
458
461
|
|
462
|
+
def get_task_config(self, task_name: str) -> dict:
|
463
|
+
"""
|
464
|
+
获取任务配置
|
465
|
+
|
466
|
+
Args:
|
467
|
+
task_name: 任务名称
|
468
|
+
|
469
|
+
Returns:
|
470
|
+
任务配置字典,如果任务不存在则返回None
|
471
|
+
"""
|
472
|
+
# 获取任务对象
|
473
|
+
task = self.get_task_by_name(task_name)
|
474
|
+
if not task:
|
475
|
+
return None
|
476
|
+
|
477
|
+
# 返回任务的配置(从Task对象的属性中提取)
|
478
|
+
return {
|
479
|
+
'auto_ack': getattr(task, 'auto_ack', True),
|
480
|
+
'queue': getattr(task, 'queue', None),
|
481
|
+
'timeout': getattr(task, 'timeout', None),
|
482
|
+
'max_retries': getattr(task, 'max_retries', 0),
|
483
|
+
'retry_delay': getattr(task, 'retry_delay', None),
|
484
|
+
}
|
485
|
+
|
459
486
|
def include_module(self, modules: list):
|
460
487
|
self.include += modules
|
461
488
|
|
462
489
|
def _task_from_fun(
|
463
|
-
self, fun, name=None, base=None, queue=None, bind=False, retry_config=None, rate_limit=None, **options
|
490
|
+
self, fun, name=None, base=None, queue=None, bind=False, retry_config=None, rate_limit=None, auto_ack=True, **options
|
464
491
|
) -> Task:
|
465
492
|
name = name or gen_task_name(fun.__name__, fun.__module__)
|
466
493
|
base = base or Task
|
@@ -480,6 +507,7 @@ class Jettask(object):
|
|
480
507
|
"queue": queue,
|
481
508
|
"retry_config": retry_config, # 存储重试配置
|
482
509
|
"rate_limit": rate_limit, # 存储限流配置
|
510
|
+
"auto_ack": auto_ack, # 存储自动ACK配置
|
483
511
|
"_decorated": True,
|
484
512
|
"__doc__": fun.__doc__,
|
485
513
|
"__module__": fun.__module__,
|
@@ -566,9 +594,25 @@ class Jettask(object):
|
|
566
594
|
retry_on_exceptions: tuple = None, # 可重试的异常类型
|
567
595
|
# 限流相关参数
|
568
596
|
rate_limit: int = None, # QPS 限制(每秒允许执行的任务数)
|
597
|
+
# ACK相关参数
|
598
|
+
auto_ack: bool = True, # 是否自动ACK(默认True)
|
569
599
|
*args,
|
570
600
|
**kwargs,
|
571
601
|
):
|
602
|
+
"""
|
603
|
+
任务装饰器 - 统一使用 TaskRouter 内部实现
|
604
|
+
|
605
|
+
Args:
|
606
|
+
name: 任务名称
|
607
|
+
queue: 队列名称
|
608
|
+
base: 基类
|
609
|
+
max_retries: 最大重试次数
|
610
|
+
retry_backoff: 是否使用指数退避
|
611
|
+
retry_backoff_max: 最大退避时间
|
612
|
+
retry_on_exceptions: 可重试的异常类型
|
613
|
+
rate_limit: 限流配置(QPS或RateLimitConfig对象)
|
614
|
+
auto_ack: 是否自动ACK(默认True)
|
615
|
+
"""
|
572
616
|
def _create_task_cls(fun):
|
573
617
|
# 将重试配置传递给_task_from_fun
|
574
618
|
retry_config = None
|
@@ -584,7 +628,19 @@ class Jettask(object):
|
|
584
628
|
exc if isinstance(exc, str) else exc.__name__
|
585
629
|
for exc in retry_on_exceptions
|
586
630
|
]
|
587
|
-
|
631
|
+
|
632
|
+
# 统一通过 _task_from_fun 创建任务,包含 auto_ack 参数
|
633
|
+
return self._task_from_fun(
|
634
|
+
fun,
|
635
|
+
name,
|
636
|
+
base,
|
637
|
+
queue,
|
638
|
+
retry_config=retry_config,
|
639
|
+
rate_limit=rate_limit,
|
640
|
+
auto_ack=auto_ack, # 传递 auto_ack
|
641
|
+
*args,
|
642
|
+
**kwargs
|
643
|
+
)
|
588
644
|
|
589
645
|
return _create_task_cls
|
590
646
|
|
@@ -681,6 +737,92 @@ class Jettask(object):
|
|
681
737
|
else:
|
682
738
|
return self._send_tasks_sync(messages)
|
683
739
|
|
740
|
+
def ack(self, ack_items: list):
|
741
|
+
"""
|
742
|
+
批量确认消息(ACK)
|
743
|
+
|
744
|
+
用于 auto_ack=False 的任务,手动批量确认消息。
|
745
|
+
这是同步方法,会在后台异步执行ACK操作。
|
746
|
+
|
747
|
+
Args:
|
748
|
+
ack_items: ACK项列表,每项可以是:
|
749
|
+
- (queue, event_id): 简单形式
|
750
|
+
- (queue, event_id, group_name): 带消费者组名
|
751
|
+
- (queue, event_id, group_name, offset): 完整形式
|
752
|
+
- dict: {'queue': ..., 'event_id': ..., 'group_name': ..., 'offset': ...}
|
753
|
+
|
754
|
+
Example:
|
755
|
+
from jettask import TaskRouter
|
756
|
+
|
757
|
+
router = TaskRouter()
|
758
|
+
|
759
|
+
@router.task(queue="batch_queue", auto_ack=False)
|
760
|
+
async def process_batch(ctx, items):
|
761
|
+
# 批量处理
|
762
|
+
results = []
|
763
|
+
ack_list = []
|
764
|
+
|
765
|
+
for item in items:
|
766
|
+
try:
|
767
|
+
result = await process_item(item)
|
768
|
+
results.append(result)
|
769
|
+
|
770
|
+
# 收集需要ACK的消息
|
771
|
+
ack_list.append((
|
772
|
+
ctx.queue,
|
773
|
+
item['event_id'],
|
774
|
+
ctx.group_name,
|
775
|
+
item.get('offset')
|
776
|
+
))
|
777
|
+
except Exception as e:
|
778
|
+
logger.error(f"Failed to process {item}: {e}")
|
779
|
+
|
780
|
+
# 批量确认成功处理的消息
|
781
|
+
ctx.app.ack(ack_list)
|
782
|
+
|
783
|
+
return results
|
784
|
+
"""
|
785
|
+
if not ack_items:
|
786
|
+
return
|
787
|
+
|
788
|
+
# 检查是否有 executor_core(Worker运行时才有)
|
789
|
+
if not hasattr(self, '_executor_core') or not self._executor_core:
|
790
|
+
logger.warning("ACK can only be called in worker context")
|
791
|
+
return
|
792
|
+
|
793
|
+
# 将ACK项添加到executor_core的pending_acks
|
794
|
+
for item in ack_items:
|
795
|
+
if isinstance(item, dict):
|
796
|
+
queue = item['queue']
|
797
|
+
event_id = item['event_id']
|
798
|
+
group_name = item.get('group_name')
|
799
|
+
offset = item.get('offset')
|
800
|
+
elif isinstance(item, (tuple, list)):
|
801
|
+
if len(item) >= 2:
|
802
|
+
queue, event_id = item[0], item[1]
|
803
|
+
group_name = item[2] if len(item) > 2 else None
|
804
|
+
offset = item[3] if len(item) > 3 else None
|
805
|
+
else:
|
806
|
+
logger.error(f"Invalid ACK item format: {item}")
|
807
|
+
continue
|
808
|
+
else:
|
809
|
+
logger.error(f"Invalid ACK item type: {type(item)}")
|
810
|
+
continue
|
811
|
+
|
812
|
+
# 添加到pending_acks
|
813
|
+
self._executor_core.pending_acks.append((queue, event_id, group_name or queue, offset))
|
814
|
+
|
815
|
+
# 检查是否需要立即刷新
|
816
|
+
if len(self._executor_core.pending_acks) >= 100:
|
817
|
+
# 创建异步任务刷新
|
818
|
+
import asyncio
|
819
|
+
try:
|
820
|
+
loop = asyncio.get_running_loop()
|
821
|
+
asyncio.create_task(self._executor_core._flush_all_buffers())
|
822
|
+
except RuntimeError:
|
823
|
+
# 不在事件循环中,稍后会自动刷新
|
824
|
+
pass
|
825
|
+
|
684
826
|
def _send_tasks_sync(self, messages: list):
|
685
827
|
"""同步发送任务"""
|
686
828
|
if not messages:
|
@@ -1860,10 +2002,17 @@ class Jettask(object):
|
|
1860
2002
|
self._mount_module()
|
1861
2003
|
|
1862
2004
|
# 收集任务列表(按队列分组)
|
2005
|
+
# 需要处理通配符队列的情况:如果 task.queue 是通配符(如 'robust_*'),
|
2006
|
+
# 则保持通配符作为键,后续动态发现时会通过通配符匹配找到对应任务
|
2007
|
+
from jettask.utils.queue_matcher import match_task_queue_to_patterns
|
2008
|
+
|
1863
2009
|
self._tasks_by_queue = {}
|
1864
2010
|
for task_name, task in self._tasks.items():
|
1865
2011
|
task_queue = task.queue or self.redis_prefix
|
1866
|
-
|
2012
|
+
|
2013
|
+
# 检查 task_queue 是否匹配 queues 中的任何一个(支持通配符)
|
2014
|
+
if match_task_queue_to_patterns(task_queue, queues):
|
2015
|
+
# 使用 task_queue(可能是通配符)作为键
|
1867
2016
|
if task_queue not in self._tasks_by_queue:
|
1868
2017
|
self._tasks_by_queue[task_queue] = []
|
1869
2018
|
self._tasks_by_queue[task_queue].append(task_name)
|
@@ -2045,14 +2194,14 @@ class Jettask(object):
|
|
2045
2194
|
logger.debug("Closing async Redis connections...")
|
2046
2195
|
|
2047
2196
|
# 关闭 EventPool 的连接
|
2048
|
-
if hasattr(self.ep, 'async_redis_client'):
|
2197
|
+
if hasattr(self.ep, 'async_redis_client') and self.ep.async_redis_client:
|
2049
2198
|
await self.ep.async_redis_client.aclose()
|
2050
2199
|
|
2051
|
-
if hasattr(self.ep, 'async_binary_redis_client'):
|
2200
|
+
if hasattr(self.ep, 'async_binary_redis_client') and self.ep.async_binary_redis_client:
|
2052
2201
|
await self.ep.async_binary_redis_client.aclose()
|
2053
2202
|
|
2054
2203
|
# 关闭 app 级别的连接
|
2055
|
-
if hasattr(self, '_async_redis'):
|
2204
|
+
if hasattr(self, '_async_redis') and self._async_redis:
|
2056
2205
|
await self._async_redis.aclose()
|
2057
2206
|
|
2058
2207
|
logger.debug("Async Redis connections closed")
|
@@ -2062,10 +2211,19 @@ class Jettask(object):
|
|
2062
2211
|
# 在新的事件循环中执行异步清理
|
2063
2212
|
try:
|
2064
2213
|
import asyncio
|
2065
|
-
|
2066
|
-
|
2067
|
-
|
2068
|
-
|
2214
|
+
# 检查是否已有运行中的事件循环
|
2215
|
+
try:
|
2216
|
+
loop = asyncio.get_running_loop()
|
2217
|
+
# 如果已有运行中的循环,直接创建 task
|
2218
|
+
asyncio.create_task(async_cleanup_redis())
|
2219
|
+
except RuntimeError:
|
2220
|
+
# 没有运行中的循环,创建新的
|
2221
|
+
cleanup_loop = asyncio.new_event_loop()
|
2222
|
+
asyncio.set_event_loop(cleanup_loop)
|
2223
|
+
try:
|
2224
|
+
cleanup_loop.run_until_complete(async_cleanup_redis())
|
2225
|
+
finally:
|
2226
|
+
cleanup_loop.close()
|
2069
2227
|
except Exception as e:
|
2070
2228
|
logger.error(f"Error in Redis cleanup: {e}", exc_info=True)
|
2071
2229
|
|
@@ -2263,7 +2421,7 @@ class Jettask(object):
|
|
2263
2421
|
)
|
2264
2422
|
|
2265
2423
|
from ..scheduler import TaskScheduler
|
2266
|
-
from ..scheduler.
|
2424
|
+
from ..scheduler.database import ScheduledTaskManager
|
2267
2425
|
|
2268
2426
|
# 创建数据库管理器
|
2269
2427
|
self.scheduler_manager = ScheduledTaskManager(db_url)
|
@@ -2281,8 +2439,9 @@ class Jettask(object):
|
|
2281
2439
|
db_manager=self.scheduler_manager,
|
2282
2440
|
**scheduler_config
|
2283
2441
|
)
|
2284
|
-
|
2285
|
-
|
2442
|
+
|
2443
|
+
# 初始化数据库管理器连接
|
2444
|
+
await self.scheduler_manager.connect()
|
2286
2445
|
logger.debug("Scheduler initialized")
|
2287
2446
|
|
2288
2447
|
async def start_scheduler(self):
|
@@ -2301,352 +2460,166 @@ class Jettask(object):
|
|
2301
2460
|
if self.scheduler:
|
2302
2461
|
self.scheduler.stop()
|
2303
2462
|
logger.debug("Scheduler stopped")
|
2304
|
-
|
2305
|
-
async def
|
2306
|
-
self,
|
2307
|
-
task_name: str,
|
2308
|
-
scheduler_id: str, # 必填参数
|
2309
|
-
queue_name: str = None,
|
2310
|
-
task_type: str = "interval",
|
2311
|
-
interval_seconds: float = None,
|
2312
|
-
cron_expression: str = None,
|
2313
|
-
task_args: list = None,
|
2314
|
-
task_kwargs: dict = None,
|
2315
|
-
next_run_time: datetime = None,
|
2316
|
-
skip_if_exists: bool = True,
|
2317
|
-
at_once: bool = True, # 是否立即保存到数据库
|
2318
|
-
**extra_params
|
2319
|
-
):
|
2463
|
+
|
2464
|
+
async def register_schedules(self, schedules):
|
2320
2465
|
"""
|
2321
|
-
|
2322
|
-
|
2466
|
+
注册定时任务(支持单个或批量)
|
2467
|
+
|
2468
|
+
这是新的统一注册方法,类似 TaskMessage 的设计模式。
|
2469
|
+
|
2323
2470
|
Args:
|
2324
|
-
|
2325
|
-
|
2326
|
-
|
2327
|
-
|
2328
|
-
|
2329
|
-
|
2330
|
-
|
2331
|
-
|
2332
|
-
|
2333
|
-
|
2334
|
-
|
2335
|
-
|
2471
|
+
schedules: ScheduledMessage 对象或列表
|
2472
|
+
|
2473
|
+
Returns:
|
2474
|
+
注册的任务数量
|
2475
|
+
|
2476
|
+
Example:
|
2477
|
+
from jettask import Schedule
|
2478
|
+
|
2479
|
+
# 1. 定义定时任务
|
2480
|
+
schedule1 = Schedule(
|
2481
|
+
scheduler_id="notify_every_30s",
|
2482
|
+
queue="notification_queue",
|
2483
|
+
interval_seconds=30,
|
2484
|
+
kwargs={"user_id": "user_123", "message": "定时提醒"}
|
2485
|
+
)
|
2486
|
+
|
2487
|
+
schedule2 = Schedule(
|
2488
|
+
scheduler_id="report_cron",
|
2489
|
+
queue="report_queue",
|
2490
|
+
cron_expression="0 9 * * *",
|
2491
|
+
description="每天生成报告"
|
2492
|
+
)
|
2493
|
+
|
2494
|
+
# 2. 批量注册
|
2495
|
+
count = await app.register_schedules([schedule1, schedule2])
|
2496
|
+
print(f"注册了 {count} 个定时任务")
|
2336
2497
|
"""
|
2498
|
+
from ..scheduler.schedule import Schedule
|
2499
|
+
from ..scheduler.models import ScheduledTask, TaskType
|
2500
|
+
|
2337
2501
|
# 自动初始化
|
2338
2502
|
await self._ensure_scheduler_initialized()
|
2339
|
-
|
2340
|
-
|
2341
|
-
|
2342
|
-
|
2343
|
-
|
2344
|
-
|
2345
|
-
|
2346
|
-
|
2347
|
-
|
2348
|
-
# 2. 如果没找到,尝试查找以task_name结尾的任务(如 "module.task_name")
|
2349
|
-
if not registered_task:
|
2350
|
-
for task_key, task_obj in self._tasks.items():
|
2351
|
-
if task_key.endswith(f".{task_name}") or task_key == task_name:
|
2352
|
-
registered_task = task_obj
|
2353
|
-
break
|
2354
|
-
|
2355
|
-
if not registered_task:
|
2356
|
-
# 任务必须已注册
|
2357
|
-
available_tasks = list(self._tasks.keys())
|
2358
|
-
error_msg = f"Task '{task_name}' not found in registered tasks.\n"
|
2359
|
-
error_msg += "All scheduled tasks must be registered with @app.task decorator.\n"
|
2360
|
-
if available_tasks:
|
2361
|
-
error_msg += f"Available tasks: {', '.join(available_tasks)}"
|
2362
|
-
raise ValueError(error_msg)
|
2363
|
-
|
2364
|
-
# 自动填充信息
|
2365
|
-
if not queue_name:
|
2366
|
-
queue_name = registered_task.queue or self.redis_prefix
|
2367
|
-
|
2368
|
-
# 使用注册时的完整任务名称(包含模块前缀)
|
2369
|
-
# 查找任务的完整注册名称
|
2370
|
-
full_task_name = None
|
2371
|
-
for task_key, task_obj in self._tasks.items():
|
2372
|
-
if task_obj == registered_task:
|
2373
|
-
full_task_name = task_key
|
2374
|
-
break
|
2375
|
-
|
2376
|
-
if not full_task_name:
|
2377
|
-
# 如果没找到,使用用户提供的名称
|
2378
|
-
full_task_name = task_name
|
2379
|
-
|
2380
|
-
# 处理 next_run_time(主要用于 once 类型)
|
2381
|
-
if task_type == "once" and extra_params.get("next_run_time"):
|
2382
|
-
# 如果在 extra_params 中,移到正确位置
|
2383
|
-
next_run_time = extra_params.pop("next_run_time", None)
|
2384
|
-
|
2385
|
-
# 移除不属于ScheduledTask的参数
|
2386
|
-
extra_params.pop("skip_if_exists", None)
|
2387
|
-
|
2388
|
-
# scheduler_id是必填参数,必须由用户提供
|
2389
|
-
if not scheduler_id:
|
2390
|
-
raise ValueError("scheduler_id is required and must be provided")
|
2391
|
-
|
2503
|
+
|
2504
|
+
# 支持单个或列表
|
2505
|
+
if isinstance(schedules, Schedule):
|
2506
|
+
schedules = [schedules]
|
2507
|
+
|
2508
|
+
if not schedules:
|
2509
|
+
return 0
|
2510
|
+
|
2392
2511
|
# 获取当前命名空间
|
2393
2512
|
namespace = 'default'
|
2394
2513
|
if self.task_center and hasattr(self.task_center, 'namespace_name'):
|
2395
2514
|
namespace = self.task_center.namespace_name
|
2396
2515
|
elif self.redis_prefix and self.redis_prefix != 'jettask':
|
2397
|
-
# 如果没有task_center,使用redis_prefix作为命名空间
|
2398
2516
|
namespace = self.redis_prefix
|
2399
|
-
|
2400
|
-
#
|
2401
|
-
|
2402
|
-
|
2403
|
-
|
2404
|
-
|
2405
|
-
|
2406
|
-
|
2407
|
-
|
2408
|
-
|
2409
|
-
|
2410
|
-
|
2411
|
-
|
2412
|
-
|
2413
|
-
|
2414
|
-
|
2415
|
-
|
2416
|
-
|
2417
|
-
|
2418
|
-
|
2419
|
-
|
2420
|
-
|
2421
|
-
|
2422
|
-
|
2423
|
-
|
2424
|
-
|
2425
|
-
|
2426
|
-
|
2427
|
-
|
2428
|
-
|
2429
|
-
|
2430
|
-
|
2431
|
-
|
2432
|
-
|
2433
|
-
|
2517
|
+
|
2518
|
+
# 转换为 ScheduledTask 对象
|
2519
|
+
tasks = []
|
2520
|
+
for schedule in schedules:
|
2521
|
+
if not isinstance(schedule, Schedule):
|
2522
|
+
raise ValueError(f"Expected Schedule, got {type(schedule)}")
|
2523
|
+
|
2524
|
+
data = schedule.to_dict()
|
2525
|
+
task = ScheduledTask(
|
2526
|
+
scheduler_id=data['scheduler_id'],
|
2527
|
+
task_type=TaskType(data['task_type']),
|
2528
|
+
queue_name=data['queue'],
|
2529
|
+
namespace=namespace,
|
2530
|
+
task_args=data['task_args'],
|
2531
|
+
task_kwargs=data['task_kwargs'],
|
2532
|
+
interval_seconds=data.get('interval_seconds'),
|
2533
|
+
cron_expression=data.get('cron_expression'),
|
2534
|
+
next_run_time=data.get('next_run_time'),
|
2535
|
+
enabled=data['enabled'],
|
2536
|
+
priority=data['priority'],
|
2537
|
+
timeout=data['timeout'],
|
2538
|
+
max_retries=data['max_retries'],
|
2539
|
+
retry_delay=data['retry_delay'],
|
2540
|
+
description=data['description'],
|
2541
|
+
tags=data['tags'],
|
2542
|
+
metadata=data['metadata']
|
2543
|
+
)
|
2544
|
+
# 保存 skip_if_exists 选项
|
2545
|
+
task._skip_if_exists = schedule.skip_if_exists
|
2546
|
+
tasks.append(task)
|
2547
|
+
|
2548
|
+
# 批量注册到数据库
|
2549
|
+
registered_count = 0
|
2550
|
+
for task in tasks:
|
2551
|
+
skip_if_exists = getattr(task, '_skip_if_exists', True)
|
2552
|
+
_, created = await self.scheduler_manager.create_or_get_task(task, skip_if_exists=skip_if_exists)
|
2553
|
+
if created:
|
2554
|
+
registered_count += 1
|
2555
|
+
logger.info(f"已注册定时任务: {task.scheduler_id} -> {task.queue_name}")
|
2556
|
+
else:
|
2557
|
+
logger.debug(f"定时任务已存在: {task.scheduler_id}")
|
2558
|
+
|
2559
|
+
return registered_count
|
2560
|
+
|
2561
|
+
async def list_schedules(self, **filters):
|
2562
|
+
"""
|
2563
|
+
列出定时任务
|
2564
|
+
|
2565
|
+
Args:
|
2566
|
+
**filters: 过滤条件(enabled, queue_name, task_type 等)
|
2567
|
+
|
2568
|
+
Returns:
|
2569
|
+
List[ScheduledTask]: 任务列表
|
2570
|
+
"""
|
2571
|
+
await self._ensure_scheduler_initialized()
|
2572
|
+
return await self.scheduler_manager.list_tasks(**filters)
|
2573
|
+
|
2574
|
+
async def remove_schedule(self, scheduler_id: str) -> bool:
|
2575
|
+
"""
|
2576
|
+
移除定时任务
|
2577
|
+
|
2578
|
+
Args:
|
2579
|
+
scheduler_id: 任务唯一标识符
|
2580
|
+
|
2581
|
+
Returns:
|
2582
|
+
bool: 是否成功移除
|
2583
|
+
"""
|
2434
2584
|
await self._ensure_scheduler_initialized()
|
2435
|
-
|
2436
|
-
# 先获取任务
|
2437
2585
|
task = await self.scheduler_manager.get_task_by_scheduler_id(scheduler_id)
|
2438
2586
|
if not task:
|
2439
2587
|
return False
|
2440
|
-
|
2441
|
-
|
2442
|
-
|
2443
|
-
if success and self.scheduler:
|
2444
|
-
# 从Redis中也移除
|
2445
|
-
await self.scheduler.loader.remove_task(task.id)
|
2446
|
-
|
2447
|
-
return success
|
2448
|
-
|
2449
|
-
async def batch_add_scheduled_tasks(
|
2450
|
-
self,
|
2451
|
-
tasks: list,
|
2452
|
-
skip_existing: bool = True
|
2453
|
-
):
|
2588
|
+
return await self.scheduler_manager.delete_task(task.id)
|
2589
|
+
|
2590
|
+
async def pause_schedule(self, scheduler_id: str) -> bool:
|
2454
2591
|
"""
|
2455
|
-
|
2456
|
-
|
2592
|
+
暂停定时任务
|
2593
|
+
|
2457
2594
|
Args:
|
2458
|
-
|
2459
|
-
|
2460
|
-
|
2595
|
+
scheduler_id: 任务唯一标识符
|
2596
|
+
|
2461
2597
|
Returns:
|
2462
|
-
|
2598
|
+
bool: 是否成功暂停
|
2463
2599
|
"""
|
2464
|
-
# 自动初始化
|
2465
2600
|
await self._ensure_scheduler_initialized()
|
2466
|
-
|
2467
|
-
|
2468
|
-
|
2469
|
-
|
2470
|
-
|
2471
|
-
|
2472
|
-
|
2473
|
-
|
2474
|
-
logger.warning("Task config missing task_name, skipping")
|
2475
|
-
continue
|
2476
|
-
|
2477
|
-
# 查找注册的任务
|
2478
|
-
registered_task = self._tasks.get(task_name)
|
2479
|
-
if not registered_task:
|
2480
|
-
for task_key, task_obj in self._tasks.items():
|
2481
|
-
if task_key.endswith(f".{task_name}") or task_key == task_name:
|
2482
|
-
registered_task = task_obj
|
2483
|
-
task_name = task_key # 使用完整名称
|
2484
|
-
break
|
2485
|
-
|
2486
|
-
if not registered_task:
|
2487
|
-
logger.warning(f"Task '{task_name}' not found in registered tasks, skipping")
|
2488
|
-
continue
|
2489
|
-
|
2490
|
-
# 准备任务参数
|
2491
|
-
queue_name = task_config.get('queue_name') or registered_task.queue or self.redis_prefix
|
2492
|
-
task_type = task_config.get('task_type', 'interval')
|
2493
|
-
|
2494
|
-
# 处理next_run_time
|
2495
|
-
next_run_time = task_config.get('next_run_time')
|
2496
|
-
if task_type == 'once' and not next_run_time:
|
2497
|
-
next_run_time = datetime.now()
|
2498
|
-
|
2499
|
-
# scheduler_id是必填参数
|
2500
|
-
scheduler_id = task_config.get('scheduler_id')
|
2501
|
-
if not scheduler_id:
|
2502
|
-
raise ValueError(f"Task config for '{task_name}' missing required scheduler_id")
|
2503
|
-
|
2504
|
-
# 获取当前命名空间
|
2505
|
-
namespace = 'default'
|
2506
|
-
if self.task_center and hasattr(self.task_center, 'namespace_name'):
|
2507
|
-
namespace = self.task_center.namespace_name
|
2508
|
-
elif self.redis_prefix and self.redis_prefix != 'jettask':
|
2509
|
-
# 如果没有task_center,使用redis_prefix作为命名空间
|
2510
|
-
namespace = self.redis_prefix
|
2511
|
-
|
2512
|
-
# 创建任务对象
|
2513
|
-
task_obj = ScheduledTask(
|
2514
|
-
scheduler_id=scheduler_id,
|
2515
|
-
task_name=task_name,
|
2516
|
-
task_type=TaskType(task_type),
|
2517
|
-
queue_name=queue_name,
|
2518
|
-
namespace=namespace, # 设置命名空间
|
2519
|
-
task_args=task_config.get('task_args', []),
|
2520
|
-
task_kwargs=task_config.get('task_kwargs', {}),
|
2521
|
-
interval_seconds=task_config.get('interval_seconds'),
|
2522
|
-
cron_expression=task_config.get('cron_expression'),
|
2523
|
-
next_run_time=next_run_time,
|
2524
|
-
enabled=task_config.get('enabled', True),
|
2525
|
-
max_retries=task_config.get('max_retries', 3),
|
2526
|
-
retry_delay=task_config.get('retry_delay', 60),
|
2527
|
-
timeout=task_config.get('timeout', 300),
|
2528
|
-
description=task_config.get('description'),
|
2529
|
-
tags=task_config.get('tags', []),
|
2530
|
-
metadata=task_config.get('metadata', {})
|
2531
|
-
)
|
2532
|
-
task_objects.append(task_obj)
|
2533
|
-
|
2534
|
-
# 批量创建
|
2535
|
-
created_tasks = await self.scheduler_manager.batch_create_tasks(task_objects, skip_existing)
|
2536
|
-
|
2537
|
-
logger.debug(f"Batch created {len(created_tasks)} scheduled tasks")
|
2538
|
-
return created_tasks
|
2539
|
-
|
2540
|
-
async def bulk_write_scheduled_tasks(self, tasks: list):
|
2601
|
+
task = await self.scheduler_manager.get_task_by_scheduler_id(scheduler_id)
|
2602
|
+
if not task:
|
2603
|
+
return False
|
2604
|
+
task.enabled = False
|
2605
|
+
await self.scheduler_manager.update_task(task)
|
2606
|
+
return True
|
2607
|
+
|
2608
|
+
async def resume_schedule(self, scheduler_id: str) -> bool:
|
2541
2609
|
"""
|
2542
|
-
|
2543
|
-
|
2544
|
-
使用示例:
|
2545
|
-
# 收集任务对象
|
2546
|
-
tasks = []
|
2547
|
-
for i in range(100):
|
2548
|
-
task = await app.add_scheduled_task(
|
2549
|
-
task_name="my_task",
|
2550
|
-
scheduler_id=f"task_{i}",
|
2551
|
-
task_type="interval",
|
2552
|
-
interval_seconds=30,
|
2553
|
-
at_once=False # 不立即保存
|
2554
|
-
)
|
2555
|
-
tasks.append(task)
|
2556
|
-
|
2557
|
-
# 批量写入
|
2558
|
-
created_tasks = await app.bulk_write_scheduled_tasks(tasks)
|
2559
|
-
|
2610
|
+
恢复定时任务
|
2611
|
+
|
2560
2612
|
Args:
|
2561
|
-
|
2562
|
-
|
2613
|
+
scheduler_id: 任务唯一标识符
|
2614
|
+
|
2563
2615
|
Returns:
|
2564
|
-
|
2616
|
+
bool: 是否成功恢复
|
2565
2617
|
"""
|
2566
|
-
# 自动初始化
|
2567
|
-
await self._ensure_scheduler_initialized()
|
2568
|
-
|
2569
|
-
if not tasks:
|
2570
|
-
return []
|
2571
|
-
|
2572
|
-
# 准备批量创建的任务列表
|
2573
|
-
task_objects = []
|
2574
|
-
for task in tasks:
|
2575
|
-
if not hasattr(task, 'scheduler_id'):
|
2576
|
-
logger.warning("Invalid task object, skipping")
|
2577
|
-
continue
|
2578
|
-
|
2579
|
-
|
2580
|
-
task_objects.append(task)
|
2581
|
-
|
2582
|
-
# 批量创建(使用第一个任务的skip_if_exists设置)
|
2583
|
-
skip_existing = getattr(tasks[0], '_skip_if_exists', True) if tasks else True
|
2584
|
-
created_tasks = await self.scheduler_manager.batch_create_tasks(task_objects, skip_existing)
|
2585
|
-
|
2586
|
-
logger.debug(f"Bulk wrote {len(created_tasks)} scheduled tasks")
|
2587
|
-
return created_tasks
|
2588
|
-
|
2589
|
-
async def list_scheduled_tasks(self, **filters):
|
2590
|
-
"""列出定时任务"""
|
2591
|
-
# 自动初始化
|
2592
|
-
await self._ensure_scheduler_initialized()
|
2593
|
-
|
2594
|
-
return await self.scheduler_manager.list_tasks(**filters)
|
2595
|
-
|
2596
|
-
async def get_scheduled_task(self, scheduler_id: str):
|
2597
|
-
"""获取定时任务详情"""
|
2598
|
-
# 自动初始化
|
2599
2618
|
await self._ensure_scheduler_initialized()
|
2600
|
-
|
2601
|
-
return await self.scheduler_manager.get_task_by_scheduler_id(scheduler_id)
|
2602
|
-
|
2603
|
-
async def pause_scheduled_task(self, scheduler_id: str):
|
2604
|
-
"""暂停/禁用定时任务"""
|
2605
|
-
# 自动初始化
|
2606
|
-
await self._ensure_scheduler_initialized()
|
2607
|
-
|
2608
|
-
# 通过scheduler_id获取任务
|
2609
2619
|
task = await self.scheduler_manager.get_task_by_scheduler_id(scheduler_id)
|
2610
|
-
|
2611
|
-
|
2612
|
-
|
2613
|
-
|
2614
|
-
|
2615
|
-
|
2616
|
-
if self.scheduler:
|
2617
|
-
await self.scheduler.loader.remove_task(task.id)
|
2618
|
-
|
2619
|
-
logger.debug(f"Task {task.id} (scheduler_id: {task.scheduler_id}) disabled")
|
2620
|
-
return True
|
2621
|
-
return False
|
2622
|
-
|
2623
|
-
async def resume_scheduled_task(self, scheduler_id: str):
|
2624
|
-
"""恢复/启用定时任务"""
|
2625
|
-
# 自动初始化
|
2626
|
-
await self._ensure_scheduler_initialized()
|
2627
|
-
|
2628
|
-
# 通过scheduler_id获取任务
|
2629
|
-
task = await self.scheduler_manager.get_task_by_scheduler_id(scheduler_id)
|
2630
|
-
|
2631
|
-
if task:
|
2632
|
-
task.enabled = True
|
2633
|
-
task.next_run_time = task.calculate_next_run_time()
|
2634
|
-
await self.scheduler_manager.update_task(task)
|
2635
|
-
|
2636
|
-
# 触发重新加载
|
2637
|
-
if self.scheduler:
|
2638
|
-
await self.scheduler.loader.load_tasks()
|
2639
|
-
|
2640
|
-
logger.debug(f"Task {task.id} (scheduler_id: {task.scheduler_id}) enabled")
|
2641
|
-
return True
|
2642
|
-
return False
|
2643
|
-
|
2644
|
-
# 别名,更直观
|
2645
|
-
async def disable_scheduled_task(self, scheduler_id: str):
|
2646
|
-
"""禁用定时任务"""
|
2647
|
-
return await self.pause_scheduled_task(scheduler_id=scheduler_id)
|
2648
|
-
|
2649
|
-
async def enable_scheduled_task(self, scheduler_id: str):
|
2650
|
-
"""启用定时任务"""
|
2651
|
-
return await self.resume_scheduled_task(scheduler_id=scheduler_id)
|
2652
|
-
|
2620
|
+
if not task:
|
2621
|
+
return False
|
2622
|
+
task.enabled = True
|
2623
|
+
await self.scheduler_manager.update_task(task)
|
2624
|
+
return True
|
2625
|
+
|