django-simpletask5 0.1.4__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/PKG-INFO +66 -5
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/README.md +65 -4
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/admin.py +125 -50
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/defaults.py +2 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/publisher.py +39 -9
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/worker_registry.py +13 -11
- django_simpletask5-0.2.1/django_simpletask5/executors/base.py +35 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/bash_script.py +11 -3
- django_simpletask5-0.2.1/django_simpletask5/executors/exceptions.py +13 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/python_script.py +10 -2
- django_simpletask5-0.2.1/django_simpletask5/executors/retry_timeout.py +66 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/simple_request.py +3 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/commands/django_simpletask_crontab.py +5 -1
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/commands/django_simpletask_executor.py +23 -2
- django_simpletask5-0.2.1/django_simpletask5/migrations/0006_alter_cronjob_options_alter_taskexecution_created_at_and_more.py +67 -0
- django_simpletask5-0.2.1/django_simpletask5/migrations/0007_rename_task_model_to_executor_class_in_stat.py +34 -0
- django_simpletask5-0.2.1/django_simpletask5/migrations/0008_taskexecution_pickup_timeout_seconds.py +20 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/models.py +20 -9
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/services/archive.py +3 -3
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5.egg-info/PKG-INFO +66 -5
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5.egg-info/SOURCES.txt +4 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/pyproject.toml +1 -1
- django_simpletask5-0.1.4/django_simpletask5/executors/base.py +0 -20
- django_simpletask5-0.1.4/django_simpletask5/executors/retry_timeout.py +0 -32
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/LICENSE +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/apps.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/cronjob_registry.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/executor_scanner.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/lock.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/message_queue.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/signals.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/dashboards.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/archive.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/loader.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/ping_pong.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/executors/status_check.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/locale/zh_Hans/LC_MESSAGES/django.mo +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/locale/zh_Hans/LC_MESSAGES/django.po +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/commands/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/commands/django_simpletask_archive.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/management/commands/django_simpletask_sync_cronjobs.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/0001_initial.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/0002_taskexecution_done_tasks_count_and_more.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/0003_alter_cronjob_options_remove_cronjob_name_and_more.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/0004_alter_cronjob_is_modified_by_user.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/0005_alter_cronjob_is_modified_by_user_help.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/migrations/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/services/__init__.py +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5.egg-info/dependency_links.txt +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5.egg-info/requires.txt +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5.egg-info/top_level.txt +0 -0
- {django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-simpletask5
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Lightweight async task execution framework for Django
|
|
5
5
|
Author-email: rRR0VrFP <rrr0vrfp@qq.com>
|
|
6
6
|
Maintainer-email: rRR0VrFP <rrr0vrfp@qq.com>
|
|
@@ -50,7 +50,8 @@ Dynamic: license-file
|
|
|
50
50
|
- **自定义事件** — 通过 `task.trigger('event_name')` 触发任意事件
|
|
51
51
|
- **灵活的执行器映射** — 不同事件可绑定不同的执行器类
|
|
52
52
|
- **队列路由** — 不同事件可路由到不同优先级队列
|
|
53
|
-
- **重试与超时** —
|
|
53
|
+
- **重试与超时** — 失败自动重试(指数退避),支持执行超时与 Pickup 超时两种检测
|
|
54
|
+
- **拒绝回退** — 执行器可通过 `RejectException` 拒绝当前任务,消息自动重新入队等待下次调度
|
|
54
55
|
- **加密字段** — 敏感数据自动加密存储
|
|
55
56
|
- **Cron 调度** — 内置 crontab 守护进程,支持代码注册与数据库覆盖
|
|
56
57
|
- **归档统计** — 已完成执行记录自动归档为加密 JSONL,并生成日统计
|
|
@@ -156,10 +157,15 @@ DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE = 'django_simpletask5.queue.high_priority'
|
|
|
156
157
|
### 执行与重试
|
|
157
158
|
|
|
158
159
|
```python
|
|
159
|
-
DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = 3
|
|
160
|
-
DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = 600
|
|
160
|
+
DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = 3 # 默认最大重试次数
|
|
161
|
+
DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = 600 # 默认执行超时(秒),任务开始执行后的超时限制
|
|
162
|
+
DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS = 3600 # 默认 Pickup 超时(秒),等待 Worker 消费的时间限制
|
|
161
163
|
```
|
|
162
164
|
|
|
165
|
+
框架区分两种超时:
|
|
166
|
+
- **执行超时**(`timeout_seconds`)— 任务进入 `running` 状态后,必须在指定时间内完成,否则被 `StatusCheckExecutor` 标记超时。适用于保护实际执行不 hang 死。
|
|
167
|
+
- **Pickup 超时**(`pickup_timeout_seconds`)— 任务创建后等待 Worker 消费的时间上限,也用于被拒绝回退后等待重新调度的保护。通常设得比执行超时更长,因为排队等待时间不可控。
|
|
168
|
+
|
|
163
169
|
### 归档
|
|
164
170
|
|
|
165
171
|
```python
|
|
@@ -177,6 +183,9 @@ DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC = True # 是否自动将代码注册的 Cro
|
|
|
177
183
|
### 安全
|
|
178
184
|
|
|
179
185
|
```python
|
|
186
|
+
# 是否启用 Python/Shell 脚本执行器(默认关闭,开启有安全风险)
|
|
187
|
+
DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS = False
|
|
188
|
+
|
|
180
189
|
# 脚本执行白名单(空列表表示不限制)
|
|
181
190
|
DJANGO_SIMPLETASK_SCRIPT_WHITELIST = ['/path/to/allowed/scripts']
|
|
182
191
|
|
|
@@ -187,6 +196,8 @@ DJANGO_SIMPLETASK_ERROR_MSG_PASSWORD = None # 错误信息加密密码
|
|
|
187
196
|
DJANGO_SIMPLETASK_CONTEXT_PASSWORD = None # 上下文数据加密密码
|
|
188
197
|
```
|
|
189
198
|
|
|
199
|
+
> 注意:`DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS` 设为 `True` 后,还需为用户/组授予 `django_simpletask5 | Cron job | Can use script executors` 权限,非超级管理员无法创建 Python/Shell 脚本执行器类型的定时任务。
|
|
200
|
+
|
|
190
201
|
## 快速开始
|
|
191
202
|
|
|
192
203
|
### 1. 定义任务模型
|
|
@@ -230,6 +241,33 @@ class OrderCreateExecutor(BaseExecutor):
|
|
|
230
241
|
return 'ok'
|
|
231
242
|
```
|
|
232
243
|
|
|
244
|
+
执行器可以通过 `executor_class` 属性覆盖超时配置:
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
class OrderCreateExecutor(BaseExecutor):
|
|
248
|
+
timeout_seconds = 300 # 执行超时 5 分钟
|
|
249
|
+
pickup_timeout_seconds = 1800 # Pickup 超时 30 分钟
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
### 拒绝回退
|
|
253
|
+
|
|
254
|
+
执行器在执行过程中遇到临时条件不满足(如资源上限),可以抛出 `RejectException` 拒绝本次执行:
|
|
255
|
+
|
|
256
|
+
```python
|
|
257
|
+
from django_simpletask5.executors.exceptions import RejectException
|
|
258
|
+
|
|
259
|
+
class MyExecutor(BaseExecutor):
|
|
260
|
+
def execute(self, execution: TaskExecution) -> str | None:
|
|
261
|
+
if not self._can_proceed():
|
|
262
|
+
raise RejectException('Resource limit reached, try again later')
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
框架捕获后会:
|
|
266
|
+
- 调用 `message.reject(requeue=True)` 将消息放回队列尾部,下次重新调度
|
|
267
|
+
- `TaskExecution` 状态恢复为 `pending`,`started_at` 清空,等待下次 Worker 消费
|
|
268
|
+
- **不消耗**重试次数
|
|
269
|
+
- 刷新 `expire_time`(使用 Pickup 超时),防止被 `StatusCheckExecutor` 误回收
|
|
270
|
+
|
|
233
271
|
### 3. 启动 Worker
|
|
234
272
|
|
|
235
273
|
默认只监听 `default` 队列,`high_priority` 队列需要单独启动 Worker 处理。
|
|
@@ -310,11 +348,34 @@ Task 模型变更 → Django 信号 → 创建 TaskExecution 并发布到消息
|
|
|
310
348
|
↓
|
|
311
349
|
Worker 消费消息 → 获取分布式锁 → 加载执行器 → 执行并保存结果
|
|
312
350
|
↓
|
|
313
|
-
|
|
351
|
+
失败时自动重试/拒绝时重新入队,完成后归档
|
|
314
352
|
```
|
|
315
353
|
|
|
316
354
|
## Releases
|
|
317
355
|
|
|
356
|
+
### 0.2.1
|
|
357
|
+
|
|
358
|
+
- **拒绝回退机制** — 新增 `RejectException`,执行器可抛出该异常拒绝当前任务,消息自动重新入队等待下次调度,不消耗重试次数
|
|
359
|
+
- **区分两种超时** — 引入 `pickup_timeout_seconds`(等待 Worker 消费超时)与 `timeout_seconds`(执行超时)两种超时配置,`expire_time` 改用 Pickup 超时计算,超时检测更精准
|
|
360
|
+
- **文档更新** — README 补充拒绝回退机制的使用说明和两种超时的详细解释
|
|
361
|
+
|
|
362
|
+
### 0.2.0
|
|
363
|
+
|
|
364
|
+
- **破坏性变更**: 移除 `Task` 模型上的自定义主键,`task_id` 不再是主键字段,改为 `unique=True` 的唯一标识字段。所有 `Task` 子模型将自动获得 Django 默认的自增 `id` 主键。已有数据库需要迁移处理。
|
|
365
|
+
- **统计维度调整**: `TaskExecutionStat` 的统计维度由 `task_model` 改为 `executor_class`,统一覆盖有模型任务和 Cron 定时任务两种场景。对应 API `_update_stats()` 按 `executor_class` 分组聚合。
|
|
366
|
+
- **Admin 优化**: CronJob 列表页移除 `executor_class` 列避免表格撑开;TaskExecutionStat 列表页新增 `executor_class` 列、`created_at` 设为只读修复详情页错误。
|
|
367
|
+
- **测试修复**: 修复 `TransactionTestCase` + `on_commit` 在 SQLite 共享内存连接下因 `TestCase` 残留原子块导致回调不执行的兼容性问题;Redis MQ / RabbitMQ E2E 集成测试全部通过。
|
|
368
|
+
|
|
369
|
+
### 0.1.5
|
|
370
|
+
|
|
371
|
+
- **安全加固** — 新增 `DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS` 配置项(默认关闭),启用后才可执行 Python/Shell 脚本;新增 `can_use_script_executors` 权限,仅有此权限或超级管理员的用户才能在 Admin 中创建脚本执行器类型的 CronJob
|
|
372
|
+
- **Bug 修复** — 修复 `RetryTimeoutExecutor` 中 lambda 晚绑定导致所有回调引用最后一个记录的问题
|
|
373
|
+
- **Admin 安全** — 操作按钮(立即执行、启用/禁用)从 GET 参数改为 POST 请求,附带 CSRF 保护
|
|
374
|
+
- **性能优化** — `RetryTimeoutExecutor` 改为批量 update + 按 ID 分发消息推送
|
|
375
|
+
- **Admin 增强** — `TaskExecutionAdmin` 新增 `executor_class` 过滤、`error_message` 搜索、批量重试失败任务、批量取消待处理任务
|
|
376
|
+
- **数据索引** — 添加 `(status, expire_time)` 和 `(is_active, next_run_time)` 复合索引;`trigger_event`、`executor_class`、`task_id`、`created_at` 添加单字段索引
|
|
377
|
+
- **代码规范** — `bash_script.py`、`python_script.py`、`simple_request.py` 补充缺失的 logger;移除废弃的 `allow_tags` 属性
|
|
378
|
+
|
|
318
379
|
### 0.1.4
|
|
319
380
|
|
|
320
381
|
- **修复打包** — `pyproject.toml` 添加 `package-data` 配置,打包时包含 locale po/mo 文件,修复 i18n 翻译不生效的问题
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
- **自定义事件** — 通过 `task.trigger('event_name')` 触发任意事件
|
|
12
12
|
- **灵活的执行器映射** — 不同事件可绑定不同的执行器类
|
|
13
13
|
- **队列路由** — 不同事件可路由到不同优先级队列
|
|
14
|
-
- **重试与超时** —
|
|
14
|
+
- **重试与超时** — 失败自动重试(指数退避),支持执行超时与 Pickup 超时两种检测
|
|
15
|
+
- **拒绝回退** — 执行器可通过 `RejectException` 拒绝当前任务,消息自动重新入队等待下次调度
|
|
15
16
|
- **加密字段** — 敏感数据自动加密存储
|
|
16
17
|
- **Cron 调度** — 内置 crontab 守护进程,支持代码注册与数据库覆盖
|
|
17
18
|
- **归档统计** — 已完成执行记录自动归档为加密 JSONL,并生成日统计
|
|
@@ -117,10 +118,15 @@ DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE = 'django_simpletask5.queue.high_priority'
|
|
|
117
118
|
### 执行与重试
|
|
118
119
|
|
|
119
120
|
```python
|
|
120
|
-
DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = 3
|
|
121
|
-
DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = 600
|
|
121
|
+
DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = 3 # 默认最大重试次数
|
|
122
|
+
DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = 600 # 默认执行超时(秒),任务开始执行后的超时限制
|
|
123
|
+
DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS = 3600 # 默认 Pickup 超时(秒),等待 Worker 消费的时间限制
|
|
122
124
|
```
|
|
123
125
|
|
|
126
|
+
框架区分两种超时:
|
|
127
|
+
- **执行超时**(`timeout_seconds`)— 任务进入 `running` 状态后,必须在指定时间内完成,否则被 `StatusCheckExecutor` 标记超时。适用于保护实际执行不 hang 死。
|
|
128
|
+
- **Pickup 超时**(`pickup_timeout_seconds`)— 任务创建后等待 Worker 消费的时间上限,也用于被拒绝回退后等待重新调度的保护。通常设得比执行超时更长,因为排队等待时间不可控。
|
|
129
|
+
|
|
124
130
|
### 归档
|
|
125
131
|
|
|
126
132
|
```python
|
|
@@ -138,6 +144,9 @@ DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC = True # 是否自动将代码注册的 Cro
|
|
|
138
144
|
### 安全
|
|
139
145
|
|
|
140
146
|
```python
|
|
147
|
+
# 是否启用 Python/Shell 脚本执行器(默认关闭,开启有安全风险)
|
|
148
|
+
DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS = False
|
|
149
|
+
|
|
141
150
|
# 脚本执行白名单(空列表表示不限制)
|
|
142
151
|
DJANGO_SIMPLETASK_SCRIPT_WHITELIST = ['/path/to/allowed/scripts']
|
|
143
152
|
|
|
@@ -148,6 +157,8 @@ DJANGO_SIMPLETASK_ERROR_MSG_PASSWORD = None # 错误信息加密密码
|
|
|
148
157
|
DJANGO_SIMPLETASK_CONTEXT_PASSWORD = None # 上下文数据加密密码
|
|
149
158
|
```
|
|
150
159
|
|
|
160
|
+
> 注意:`DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS` 设为 `True` 后,还需为用户/组授予 `django_simpletask5 | Cron job | Can use script executors` 权限,非超级管理员无法创建 Python/Shell 脚本执行器类型的定时任务。
|
|
161
|
+
|
|
151
162
|
## 快速开始
|
|
152
163
|
|
|
153
164
|
### 1. 定义任务模型
|
|
@@ -191,6 +202,33 @@ class OrderCreateExecutor(BaseExecutor):
|
|
|
191
202
|
return 'ok'
|
|
192
203
|
```
|
|
193
204
|
|
|
205
|
+
执行器可以通过 `executor_class` 属性覆盖超时配置:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
class OrderCreateExecutor(BaseExecutor):
|
|
209
|
+
timeout_seconds = 300 # 执行超时 5 分钟
|
|
210
|
+
pickup_timeout_seconds = 1800 # Pickup 超时 30 分钟
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### 拒绝回退
|
|
214
|
+
|
|
215
|
+
执行器在执行过程中遇到临时条件不满足(如资源上限),可以抛出 `RejectException` 拒绝本次执行:
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
from django_simpletask5.executors.exceptions import RejectException
|
|
219
|
+
|
|
220
|
+
class MyExecutor(BaseExecutor):
|
|
221
|
+
def execute(self, execution: TaskExecution) -> str | None:
|
|
222
|
+
if not self._can_proceed():
|
|
223
|
+
raise RejectException('Resource limit reached, try again later')
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
框架捕获后会:
|
|
227
|
+
- 调用 `message.reject(requeue=True)` 将消息放回队列尾部,下次重新调度
|
|
228
|
+
- `TaskExecution` 状态恢复为 `pending`,`started_at` 清空,等待下次 Worker 消费
|
|
229
|
+
- **不消耗**重试次数
|
|
230
|
+
- 刷新 `expire_time`(使用 Pickup 超时),防止被 `StatusCheckExecutor` 误回收
|
|
231
|
+
|
|
194
232
|
### 3. 启动 Worker
|
|
195
233
|
|
|
196
234
|
默认只监听 `default` 队列,`high_priority` 队列需要单独启动 Worker 处理。
|
|
@@ -271,11 +309,34 @@ Task 模型变更 → Django 信号 → 创建 TaskExecution 并发布到消息
|
|
|
271
309
|
↓
|
|
272
310
|
Worker 消费消息 → 获取分布式锁 → 加载执行器 → 执行并保存结果
|
|
273
311
|
↓
|
|
274
|
-
|
|
312
|
+
失败时自动重试/拒绝时重新入队,完成后归档
|
|
275
313
|
```
|
|
276
314
|
|
|
277
315
|
## Releases
|
|
278
316
|
|
|
317
|
+
### 0.2.1
|
|
318
|
+
|
|
319
|
+
- **拒绝回退机制** — 新增 `RejectException`,执行器可抛出该异常拒绝当前任务,消息自动重新入队等待下次调度,不消耗重试次数
|
|
320
|
+
- **区分两种超时** — 引入 `pickup_timeout_seconds`(等待 Worker 消费超时)与 `timeout_seconds`(执行超时)两种超时配置,`expire_time` 改用 Pickup 超时计算,超时检测更精准
|
|
321
|
+
- **文档更新** — README 补充拒绝回退机制的使用说明和两种超时的详细解释
|
|
322
|
+
|
|
323
|
+
### 0.2.0
|
|
324
|
+
|
|
325
|
+
- **破坏性变更**: 移除 `Task` 模型上的自定义主键,`task_id` 不再是主键字段,改为 `unique=True` 的唯一标识字段。所有 `Task` 子模型将自动获得 Django 默认的自增 `id` 主键。已有数据库需要迁移处理。
|
|
326
|
+
- **统计维度调整**: `TaskExecutionStat` 的统计维度由 `task_model` 改为 `executor_class`,统一覆盖有模型任务和 Cron 定时任务两种场景。对应 API `_update_stats()` 按 `executor_class` 分组聚合。
|
|
327
|
+
- **Admin 优化**: CronJob 列表页移除 `executor_class` 列避免表格撑开;TaskExecutionStat 列表页新增 `executor_class` 列、`created_at` 设为只读修复详情页错误。
|
|
328
|
+
- **测试修复**: 修复 `TransactionTestCase` + `on_commit` 在 SQLite 共享内存连接下因 `TestCase` 残留原子块导致回调不执行的兼容性问题;Redis MQ / RabbitMQ E2E 集成测试全部通过。
|
|
329
|
+
|
|
330
|
+
### 0.1.5
|
|
331
|
+
|
|
332
|
+
- **安全加固** — 新增 `DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS` 配置项(默认关闭),启用后才可执行 Python/Shell 脚本;新增 `can_use_script_executors` 权限,仅有此权限或超级管理员的用户才能在 Admin 中创建脚本执行器类型的 CronJob
|
|
333
|
+
- **Bug 修复** — 修复 `RetryTimeoutExecutor` 中 lambda 晚绑定导致所有回调引用最后一个记录的问题
|
|
334
|
+
- **Admin 安全** — 操作按钮(立即执行、启用/禁用)从 GET 参数改为 POST 请求,附带 CSRF 保护
|
|
335
|
+
- **性能优化** — `RetryTimeoutExecutor` 改为批量 update + 按 ID 分发消息推送
|
|
336
|
+
- **Admin 增强** — `TaskExecutionAdmin` 新增 `executor_class` 过滤、`error_message` 搜索、批量重试失败任务、批量取消待处理任务
|
|
337
|
+
- **数据索引** — 添加 `(status, expire_time)` 和 `(is_active, next_run_time)` 复合索引;`trigger_event`、`executor_class`、`task_id`、`created_at` 添加单字段索引
|
|
338
|
+
- **代码规范** — `bash_script.py`、`python_script.py`、`simple_request.py` 补充缺失的 logger;移除废弃的 `allow_tags` 属性
|
|
339
|
+
|
|
279
340
|
### 0.1.4
|
|
280
341
|
|
|
281
342
|
- **修复打包** — `pyproject.toml` 添加 `package-data` 配置,打包时包含 locale po/mo 文件,修复 i18n 翻译不生效的问题
|
|
@@ -2,8 +2,9 @@ import json
|
|
|
2
2
|
|
|
3
3
|
from django import forms
|
|
4
4
|
from django.contrib import admin
|
|
5
|
-
from django.http import HttpResponseRedirect
|
|
6
|
-
from django.urls import reverse
|
|
5
|
+
from django.http import HttpResponseRedirect, Http404
|
|
6
|
+
from django.urls import path, reverse
|
|
7
|
+
from django.utils.html import format_html, mark_safe
|
|
7
8
|
from django.utils.translation import gettext_lazy as _
|
|
8
9
|
from django_checkbox_normalize.admin import DjangoCheckboxNormalizeAdmin
|
|
9
10
|
from django_tabbed_changeform_admin.admin import DjangoTabbedChangeformAdmin
|
|
@@ -15,6 +16,11 @@ from django_simpletask5.core.executor_scanner import (
|
|
|
15
16
|
get_executor_schema_map,
|
|
16
17
|
)
|
|
17
18
|
|
|
19
|
+
SCRIPT_EXECUTOR_CLASSES = {
|
|
20
|
+
'django_simpletask5.executors.bash_script.BashScriptExecutor',
|
|
21
|
+
'django_simpletask5.executors.python_script.PythonScriptExecutor',
|
|
22
|
+
}
|
|
23
|
+
|
|
18
24
|
|
|
19
25
|
class ExecutorClassWidget(forms.Select):
|
|
20
26
|
def __init__(self, attrs=None):
|
|
@@ -58,6 +64,7 @@ class CronJobForm(forms.ModelForm):
|
|
|
58
64
|
)
|
|
59
65
|
|
|
60
66
|
def __init__(self, *args, **kwargs):
|
|
67
|
+
self.request = kwargs.pop('request', None)
|
|
61
68
|
super().__init__(*args, **kwargs)
|
|
62
69
|
if self.instance and self.instance.pk:
|
|
63
70
|
self.fields['uid'].disabled = True
|
|
@@ -75,6 +82,16 @@ class CronJobForm(forms.ModelForm):
|
|
|
75
82
|
except Exception:
|
|
76
83
|
pass
|
|
77
84
|
|
|
85
|
+
def clean_executor_class(self):
|
|
86
|
+
executor_class = self.cleaned_data.get("executor_class", "")
|
|
87
|
+
if executor_class in SCRIPT_EXECUTOR_CLASSES:
|
|
88
|
+
if self.request and not (self.request.user.is_superuser or self.request.user.has_perm('django_simpletask5.can_use_script_executors')):
|
|
89
|
+
raise forms.ValidationError(
|
|
90
|
+
_('You do not have permission to use script executors (Python/Shell). '
|
|
91
|
+
'Only superusers and users with the "can_use_script_executors" permission can do so.')
|
|
92
|
+
)
|
|
93
|
+
return executor_class
|
|
94
|
+
|
|
78
95
|
def clean_context(self):
|
|
79
96
|
raw = self.cleaned_data.get("context")
|
|
80
97
|
if not raw:
|
|
@@ -125,7 +142,6 @@ class TaskExecutionAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdm
|
|
|
125
142
|
|
|
126
143
|
@admin.display(description=_("Execution ID"))
|
|
127
144
|
def short_execution_id(self, obj):
|
|
128
|
-
from django.utils.html import format_html
|
|
129
145
|
full = str(obj.execution_id)
|
|
130
146
|
short = full[:8]
|
|
131
147
|
copied_text = _("Copied!")
|
|
@@ -136,8 +152,8 @@ class TaskExecutionAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdm
|
|
|
136
152
|
'setTimeout(function(){{t.textContent=o;}},800);}}.bind(this)).catch(function(){{}})">{}</span>',
|
|
137
153
|
full, full, copied_text, short
|
|
138
154
|
)
|
|
139
|
-
list_filter = ["status", "trigger_event", "created_at"]
|
|
140
|
-
search_fields = ["execution_id", "task_id", "executor_class"]
|
|
155
|
+
list_filter = ["status", "trigger_event", "executor_class", "created_at"]
|
|
156
|
+
search_fields = ["execution_id", "task_id", "executor_class", "error_message"]
|
|
141
157
|
readonly_fields = [
|
|
142
158
|
"execution_id", "task_id", "task_model", "trigger_event", "executor_class", "status",
|
|
143
159
|
"result", "error_message", "context", "total_tasks_count", "done_tasks_count",
|
|
@@ -145,6 +161,7 @@ class TaskExecutionAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdm
|
|
|
145
161
|
"started_at", "finished_at", "created_at", "updated_at",
|
|
146
162
|
]
|
|
147
163
|
ordering = ["-created_at"]
|
|
164
|
+
actions = ["retry_selected", "cancel_selected"]
|
|
148
165
|
|
|
149
166
|
def has_add_permission(self, request):
|
|
150
167
|
return False
|
|
@@ -185,6 +202,23 @@ class TaskExecutionAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdm
|
|
|
185
202
|
def retry_info(self, obj):
|
|
186
203
|
return f"{obj.retry_count}/{obj.max_retries}"
|
|
187
204
|
|
|
205
|
+
@admin.action(description=_("Retry selected failed tasks"))
|
|
206
|
+
def retry_selected(self, request, queryset):
|
|
207
|
+
from django_simpletask5.core.publisher import _publish_message
|
|
208
|
+
count = 0
|
|
209
|
+
for exec_record in queryset.filter(status__in=['failed', 'timeout']):
|
|
210
|
+
exec_record.status = 'retry'
|
|
211
|
+
exec_record.retry_count += 1
|
|
212
|
+
exec_record.save(update_fields=['status', 'retry_count', 'updated_at'])
|
|
213
|
+
_publish_message(exec_record)
|
|
214
|
+
count += 1
|
|
215
|
+
self.message_user(request, _("%d execution(s) have been re-queued.") % count)
|
|
216
|
+
|
|
217
|
+
@admin.action(description=_("Cancel selected pending tasks"))
|
|
218
|
+
def cancel_selected(self, request, queryset):
|
|
219
|
+
updated = queryset.filter(status='pending').update(status='failed')
|
|
220
|
+
self.message_user(request, _("%d execution(s) have been cancelled.") % updated)
|
|
221
|
+
|
|
188
222
|
|
|
189
223
|
@admin.register(CronJob)
|
|
190
224
|
class CronJobAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdmin):
|
|
@@ -192,7 +226,6 @@ class CronJobAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdmin):
|
|
|
192
226
|
list_display = [
|
|
193
227
|
"display_name",
|
|
194
228
|
"cron_expression",
|
|
195
|
-
"executor_class",
|
|
196
229
|
"is_active",
|
|
197
230
|
"is_modified_by_user",
|
|
198
231
|
"updated_at",
|
|
@@ -228,62 +261,102 @@ class CronJobAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdmin):
|
|
|
228
261
|
def get_queryset(self, request):
|
|
229
262
|
return super().get_queryset(request)
|
|
230
263
|
|
|
231
|
-
def
|
|
232
|
-
|
|
233
|
-
|
|
264
|
+
def get_urls(self):
|
|
265
|
+
urls = super().get_urls()
|
|
266
|
+
custom_urls = [
|
|
267
|
+
path(
|
|
268
|
+
'<path:object_id>/run/',
|
|
269
|
+
self.admin_site.admin_view(self._run_cronjob_view),
|
|
270
|
+
name='django_simpletask5_cronjob_run',
|
|
271
|
+
),
|
|
272
|
+
path(
|
|
273
|
+
'<path:object_id>/toggle-active/',
|
|
274
|
+
self.admin_site.admin_view(self._toggle_active_view),
|
|
275
|
+
name='django_simpletask5_cronjob_toggle_active',
|
|
276
|
+
),
|
|
277
|
+
]
|
|
278
|
+
return custom_urls + urls
|
|
234
279
|
|
|
235
|
-
|
|
280
|
+
def _run_cronjob_view(self, request, object_id):
|
|
281
|
+
if request.method != 'POST':
|
|
282
|
+
return HttpResponseRedirect(
|
|
283
|
+
reverse('admin:django_simpletask5_cronjob_changelist')
|
|
284
|
+
)
|
|
285
|
+
try:
|
|
286
|
+
cronjob = CronJob.objects.get(pk=object_id)
|
|
287
|
+
from django_simpletask5.management.commands.django_simpletask_crontab import Command as CrontabCommand
|
|
288
|
+
cmd = CrontabCommand()
|
|
289
|
+
cmd._trigger_cronjob(cronjob)
|
|
290
|
+
self.message_user(request, _('Cronjob "%s" triggered.') % cronjob.display_name)
|
|
291
|
+
except CronJob.DoesNotExist:
|
|
292
|
+
raise Http404()
|
|
293
|
+
return HttpResponseRedirect(
|
|
294
|
+
reverse('admin:django_simpletask5_cronjob_changelist')
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
def _toggle_active_view(self, request, object_id):
|
|
298
|
+
if request.method != 'POST':
|
|
299
|
+
return HttpResponseRedirect(
|
|
300
|
+
reverse('admin:django_simpletask5_cronjob_changelist')
|
|
301
|
+
)
|
|
302
|
+
try:
|
|
303
|
+
cronjob = CronJob.objects.get(pk=object_id)
|
|
304
|
+
cronjob.is_active = not cronjob.is_active
|
|
305
|
+
cronjob.save(update_fields=['is_active'])
|
|
306
|
+
status = _('activated') if cronjob.is_active else _('deactivated')
|
|
307
|
+
self.message_user(request, _('Cronjob "%s" %s.') % (cronjob.display_name, status))
|
|
308
|
+
except CronJob.DoesNotExist:
|
|
309
|
+
raise Http404()
|
|
310
|
+
return HttpResponseRedirect(
|
|
311
|
+
reverse('admin:django_simpletask5_cronjob_changelist')
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
def action_buttons(self, obj):
|
|
315
|
+
run_url = reverse('admin:django_simpletask5_cronjob_run', args=[obj.pk])
|
|
316
|
+
toggle_url = reverse('admin:django_simpletask5_cronjob_toggle_active', args=[obj.pk])
|
|
317
|
+
csrf_token = '"{{ csrf_token }}"'
|
|
236
318
|
|
|
237
319
|
buttons = []
|
|
238
320
|
buttons.append(
|
|
239
|
-
f'<
|
|
240
|
-
f'
|
|
241
|
-
f'<
|
|
321
|
+
f'<form style="display:inline" method="post" action="{run_url}">'
|
|
322
|
+
f'<input type="hidden" name="csrfmiddlewaretoken" value={csrf_token}>'
|
|
323
|
+
f'<button type="submit" style="background:#17a2b8;color:#fff;padding:3px 8px;border-radius:3px;border:none;cursor:pointer;margin-right:4px;font-size:12px" title="{_("Execute now")}">'
|
|
324
|
+
f'<i class="ri-play-line"></i> {_("Run")}</button></form>'
|
|
242
325
|
)
|
|
243
326
|
if obj.is_active:
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
)
|
|
327
|
+
btn_style = "background:#ffc107;color:#000"
|
|
328
|
+
btn_title = _("Deactivate")
|
|
329
|
+
btn_icon = "ri-pause-line"
|
|
330
|
+
btn_label = _("Deactivate")
|
|
249
331
|
else:
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
332
|
+
btn_style = "background:#28a745;color:#fff"
|
|
333
|
+
btn_title = _("Activate")
|
|
334
|
+
btn_icon = "ri-play-circle-line"
|
|
335
|
+
btn_label = _("Activate")
|
|
336
|
+
buttons.append(
|
|
337
|
+
f'<form style="display:inline" method="post" action="{toggle_url}">'
|
|
338
|
+
f'<input type="hidden" name="csrfmiddlewaretoken" value={csrf_token}>'
|
|
339
|
+
f'<button type="submit" style="{btn_style};padding:3px 8px;border-radius:3px;border:none;cursor:pointer;margin-right:4px;font-size:12px" title="{btn_title}">'
|
|
340
|
+
f'<i class="{btn_icon}"></i> {btn_label}</button></form>'
|
|
341
|
+
)
|
|
255
342
|
delete_url = reverse('admin:django_simpletask5_cronjob_delete', args=[obj.pk])
|
|
256
343
|
buttons.append(
|
|
257
344
|
f'<a class="button" style="background:#dc3545;color:#fff;padding:3px 8px;border-radius:3px;text-decoration:none;margin-right:4px;font-size:12px" '
|
|
258
|
-
f'href="{delete_url}" title="
|
|
259
|
-
f'<i class="ri-delete-bin-line"></i>
|
|
345
|
+
f'href="{delete_url}" title="{_("Delete")}">'
|
|
346
|
+
f'<i class="ri-delete-bin-line"></i> {_("Delete")}</a>'
|
|
260
347
|
)
|
|
261
|
-
return
|
|
348
|
+
return mark_safe(''.join(buttons))
|
|
262
349
|
|
|
263
|
-
action_buttons.short_description = _("
|
|
264
|
-
action_buttons.allow_tags = True
|
|
350
|
+
action_buttons.short_description = _("Actions")
|
|
265
351
|
|
|
266
|
-
def
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
from django_simpletask5.management.commands.django_simpletask_crontab import Command as CrontabCommand
|
|
275
|
-
cmd = CrontabCommand()
|
|
276
|
-
cmd._trigger_cronjob(cronjob)
|
|
277
|
-
self.message_user(request, f'Cronjob "{cronjob.display_name}" triggered.')
|
|
278
|
-
elif action == 'toggle-active':
|
|
279
|
-
cronjob.is_active = not cronjob.is_active
|
|
280
|
-
cronjob.save(update_fields=['is_active'])
|
|
281
|
-
status = 'activated' if cronjob.is_active else 'deactivated'
|
|
282
|
-
self.message_user(request, f'Cronjob "{cronjob.display_name}" {status}.')
|
|
283
|
-
except CronJob.DoesNotExist:
|
|
284
|
-
self.message_user(request, f'Cronjob not found.', level='ERROR')
|
|
285
|
-
return HttpResponseRedirect(request.path)
|
|
286
|
-
return super().changelist_view(request, extra_context=extra_context)
|
|
352
|
+
def get_form(self, request, obj=None, change=False, **kwargs):
|
|
353
|
+
form_class = super().get_form(request, obj=obj, change=change, **kwargs)
|
|
354
|
+
|
|
355
|
+
class _CronJobFormWithRequest(form_class):
|
|
356
|
+
def __init__(self, *args, **kwargs):
|
|
357
|
+
kwargs['request'] = request
|
|
358
|
+
super().__init__(*args, **kwargs)
|
|
359
|
+
return _CronJobFormWithRequest
|
|
287
360
|
|
|
288
361
|
def save_model(self, request, obj, form, change):
|
|
289
362
|
if change:
|
|
@@ -344,6 +417,7 @@ class CronJobAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdmin):
|
|
|
344
417
|
class TaskExecutionStatAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangeformAdmin):
|
|
345
418
|
list_display = [
|
|
346
419
|
"date",
|
|
420
|
+
"executor_class",
|
|
347
421
|
"trigger_event",
|
|
348
422
|
"total_count",
|
|
349
423
|
"success_count",
|
|
@@ -351,11 +425,12 @@ class TaskExecutionStatAdmin(DjangoCheckboxNormalizeAdmin, DjangoTabbedChangefor
|
|
|
351
425
|
"avg_duration_seconds",
|
|
352
426
|
]
|
|
353
427
|
list_filter = ["date", "trigger_event"]
|
|
428
|
+
readonly_fields = ["created_at"]
|
|
354
429
|
ordering = ["-date"]
|
|
355
430
|
|
|
356
431
|
fieldsets = [
|
|
357
432
|
(None, {
|
|
358
|
-
"fields": ["date", "
|
|
433
|
+
"fields": ["date", "executor_class", "trigger_event", "created_at"],
|
|
359
434
|
"classes": ["tab-basic"],
|
|
360
435
|
}),
|
|
361
436
|
(None, {
|
|
@@ -4,6 +4,7 @@ DJANGO_SIMPLETASK_LOCK_TIMEOUT = getattr(settings, 'DJANGO_SIMPLETASK_LOCK_TIMEO
|
|
|
4
4
|
|
|
5
5
|
DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES', 3)
|
|
6
6
|
DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS', 600)
|
|
7
|
+
DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS', 3600)
|
|
7
8
|
|
|
8
9
|
DJANGO_SIMPLETASK_ARCHIVE_PATH = getattr(settings, 'DJANGO_SIMPLETASK_ARCHIVE_PATH', 'django_simpletask5_archives')
|
|
9
10
|
DJANGO_SIMPLETASK_ARCHIVE_SALT = getattr(settings, 'DJANGO_SIMPLETASK_ARCHIVE_SALT', 'django-simpletask5-archive')
|
|
@@ -14,6 +15,7 @@ DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC = getattr(settings, 'DJANGO_SIMPLETASK_CRONJ
|
|
|
14
15
|
DJANGO_SIMPLETASK_DEFAULT_QUEUE = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_QUEUE', 'django_simpletask5.queue.default')
|
|
15
16
|
DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE = getattr(settings, 'DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE', 'django_simpletask5.queue.high_priority')
|
|
16
17
|
|
|
18
|
+
DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS = getattr(settings, 'DJANGO_SIMPLETASK_ENABLE_SCRIPT_EXECUTORS', False)
|
|
17
19
|
DJANGO_SIMPLETASK_SCRIPT_WHITELIST = getattr(settings, 'DJANGO_SIMPLETASK_SCRIPT_WHITELIST', [])
|
|
18
20
|
|
|
19
21
|
DJANGO_SIMPLETASK_FIELD_CIPHER_CLASS = getattr(settings, 'DJANGO_SIMPLETASK_FIELD_CIPHER_CLASS', None)
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
+
import time
|
|
3
4
|
|
|
4
5
|
from django.db import transaction
|
|
6
|
+
from django.utils import timezone
|
|
5
7
|
|
|
6
8
|
from django_simpletask5.models import TaskExecution
|
|
7
|
-
from django_simpletask5.core.defaults import DJANGO_SIMPLETASK_DEFAULT_QUEUE, DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS
|
|
9
|
+
from django_simpletask5.core.defaults import DJANGO_SIMPLETASK_DEFAULT_QUEUE, DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS, DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS
|
|
8
10
|
from django_simpletask5.executors.loader import load_executor_class
|
|
9
11
|
|
|
10
12
|
logger = logging.getLogger(__name__)
|
|
@@ -20,6 +22,16 @@ def _resolve_timeout_seconds(executor_class_path: str) -> int:
|
|
|
20
22
|
return DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS
|
|
21
23
|
|
|
22
24
|
|
|
25
|
+
def _resolve_pickup_timeout_seconds(executor_class_path: str) -> int:
|
|
26
|
+
try:
|
|
27
|
+
cls = load_executor_class(executor_class_path)
|
|
28
|
+
if cls.pickup_timeout_seconds is not None:
|
|
29
|
+
return cls.pickup_timeout_seconds
|
|
30
|
+
except Exception:
|
|
31
|
+
logger.warning('Failed to load executor %s for pickup timeout resolution', executor_class_path)
|
|
32
|
+
return DJANGO_SIMPLETASK_DEFAULT_PICKUP_TIMEOUT_SECONDS
|
|
33
|
+
|
|
34
|
+
|
|
23
35
|
def create_and_publish(task, trigger_event, extra_context=None):
|
|
24
36
|
executor_class_path = task.get_executor_class_path(event=trigger_event)
|
|
25
37
|
queue_name = task.get_queue_name(event=trigger_event)
|
|
@@ -41,6 +53,7 @@ def create_and_publish(task, trigger_event, extra_context=None):
|
|
|
41
53
|
executor_class=executor_class_path,
|
|
42
54
|
status='pending',
|
|
43
55
|
timeout_seconds=_resolve_timeout_seconds(executor_class_path),
|
|
56
|
+
pickup_timeout_seconds=_resolve_pickup_timeout_seconds(executor_class_path),
|
|
44
57
|
context=json.dumps(serialized_context, default=str) if serialized_context else None,
|
|
45
58
|
)
|
|
46
59
|
execution.save()
|
|
@@ -58,16 +71,33 @@ def _publish_message(execution, routing_key=None):
|
|
|
58
71
|
'executor_class': execution.executor_class,
|
|
59
72
|
}
|
|
60
73
|
|
|
74
|
+
from django_simpletask5.core.message_queue import get_message_queue
|
|
75
|
+
|
|
76
|
+
for attempt in range(1, 4):
|
|
77
|
+
try:
|
|
78
|
+
mq = get_message_queue()
|
|
79
|
+
mq.publish(
|
|
80
|
+
routing_key=routing_key or DJANGO_SIMPLETASK_DEFAULT_QUEUE,
|
|
81
|
+
message=message,
|
|
82
|
+
)
|
|
83
|
+
logger.debug('Published message for execution %s', execution.execution_id)
|
|
84
|
+
return
|
|
85
|
+
except Exception:
|
|
86
|
+
if attempt < 3:
|
|
87
|
+
logger.warning('Publish attempt %d/3 failed for execution %s, retrying...', attempt, execution.execution_id)
|
|
88
|
+
time.sleep(1)
|
|
89
|
+
else:
|
|
90
|
+
logger.exception('Failed to publish message for execution %s after 3 attempts', execution.execution_id)
|
|
91
|
+
|
|
61
92
|
try:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
logger.debug('Published message for execution %s', execution.execution_id)
|
|
93
|
+
execution.refresh_from_db()
|
|
94
|
+
if execution.status not in ('success', 'failed', 'timeout'):
|
|
95
|
+
execution.status = 'failed'
|
|
96
|
+
execution.error_message = 'Failed to publish message to queue after 3 attempts'
|
|
97
|
+
execution.finished_at = timezone.now()
|
|
98
|
+
execution.save(update_fields=['status', 'error_message', 'finished_at', 'updated_at'])
|
|
69
99
|
except Exception:
|
|
70
|
-
logger.exception('Failed to
|
|
100
|
+
logger.exception('Failed to mark execution %s as failed after publish failure', execution.execution_id)
|
|
71
101
|
|
|
72
102
|
|
|
73
103
|
def _serialize_context(context):
|
{django_simpletask5-0.1.4 → django_simpletask5-0.2.1}/django_simpletask5/core/worker_registry.py
RENAMED
|
@@ -12,6 +12,7 @@ logger = logging.getLogger(__name__)
|
|
|
12
12
|
WORKER_TTL = 300
|
|
13
13
|
|
|
14
14
|
_REDIS_UNAVAILABLE = False
|
|
15
|
+
_redis_lock = threading.Lock()
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def _get_redis_client():
|
|
@@ -30,17 +31,18 @@ def _get_redis_client():
|
|
|
30
31
|
|
|
31
32
|
def _safe_redis_call(fn, default=None, log_level=logging.WARNING):
|
|
32
33
|
global _REDIS_UNAVAILABLE
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
_REDIS_UNAVAILABLE
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
34
|
+
with _redis_lock:
|
|
35
|
+
try:
|
|
36
|
+
result = fn()
|
|
37
|
+
if _REDIS_UNAVAILABLE:
|
|
38
|
+
_REDIS_UNAVAILABLE = False
|
|
39
|
+
logger.info('Redis connection restored')
|
|
40
|
+
return result
|
|
41
|
+
except Exception as e:
|
|
42
|
+
if not _REDIS_UNAVAILABLE:
|
|
43
|
+
_REDIS_UNAVAILABLE = True
|
|
44
|
+
logger.log(log_level, 'Redis operation failed: %s', e)
|
|
45
|
+
return default
|
|
44
46
|
|
|
45
47
|
|
|
46
48
|
def _worker_key(pid, tid):
|