simple-module-background-tasks 0.0.32__tar.gz → 0.0.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/PKG-INFO +5 -5
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/constants.py +17 -2
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/schemas.py +21 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/deps.py +2 -2
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/api_admin.py +43 -8
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/views.py +62 -20
- simple_module_background_tasks-0.0.34/background_tasks/filters.py +69 -0
- simple_module_background_tasks-0.0.34/background_tasks/health.py +58 -0
- simple_module_background_tasks-0.0.34/background_tasks/locales/en.json +152 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/module.py +66 -2
- simple_module_background_tasks-0.0.34/background_tasks/pages/Detail.tsx +129 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/Index.tsx +232 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/Workers.tsx +115 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/DetailFacts.tsx +118 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/ExecutionRow.tsx +68 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/ExecutionsTable.tsx +120 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/PayloadCards.tsx +50 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/RetryAllDialog.tsx +76 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/RetryConfirmDialog.tsx +93 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/StatusPill.tsx +30 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/StatusStrip.tsx +128 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/TaskFilters.tsx +93 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/TracebackCard.tsx +81 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/WorkerCard.tsx +102 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/components/WorkerEmptyStates.tsx +79 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/constants.ts +255 -0
- simple_module_background_tasks-0.0.34/background_tasks/pages/retry.ts +94 -0
- simple_module_background_tasks-0.0.34/background_tasks/retry_service.py +220 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/service.py +77 -51
- simple_module_background_tasks-0.0.34/background_tasks/services.py +37 -0
- simple_module_background_tasks-0.0.34/background_tasks/settings.py +170 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/worker_inspector.py +72 -2
- simple_module_background_tasks-0.0.34/background_tasks/workers_state.py +31 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/pyproject.toml +5 -5
- simple_module_background_tasks-0.0.34/tests/conftest.py +113 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_admin_api.py +29 -0
- simple_module_background_tasks-0.0.34/tests/test_background_tasks_settings.py +27 -0
- simple_module_background_tasks-0.0.34/tests/test_bg_redis_health.py +121 -0
- simple_module_background_tasks-0.0.34/tests/test_bg_redis_url.py +106 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_service.py +19 -12
- simple_module_background_tasks-0.0.34/tests/test_bg_settings_env.py +118 -0
- simple_module_background_tasks-0.0.34/tests/test_detail_props.py +78 -0
- simple_module_background_tasks-0.0.34/tests/test_index_filters.py +186 -0
- simple_module_background_tasks-0.0.34/tests/test_retry_failed_bulk.py +150 -0
- simple_module_background_tasks-0.0.34/tests/test_retry_failed_guards.py +155 -0
- simple_module_background_tasks-0.0.34/tests/test_retry_sweep_ordering.py +200 -0
- simple_module_background_tasks-0.0.34/tests/test_workers_uptime.py +241 -0
- simple_module_background_tasks-0.0.34/tests-js/task-formatters.test.ts +150 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tsconfig.json +6 -1
- simple_module_background_tasks-0.0.32/background_tasks/locales/en.json +0 -76
- simple_module_background_tasks-0.0.32/background_tasks/pages/Detail.tsx +0 -188
- simple_module_background_tasks-0.0.32/background_tasks/pages/Index.tsx +0 -240
- simple_module_background_tasks-0.0.32/background_tasks/pages/Workers.tsx +0 -186
- simple_module_background_tasks-0.0.32/background_tasks/pages/components/ExecutionRow.tsx +0 -82
- simple_module_background_tasks-0.0.32/background_tasks/pages/components/RetryConfirmDialog.tsx +0 -81
- simple_module_background_tasks-0.0.32/background_tasks/pages/components/StatusStrip.tsx +0 -66
- simple_module_background_tasks-0.0.32/background_tasks/pages/constants.ts +0 -70
- simple_module_background_tasks-0.0.32/background_tasks/pages/retry.ts +0 -45
- simple_module_background_tasks-0.0.32/background_tasks/services.py +0 -25
- simple_module_background_tasks-0.0.32/background_tasks/settings.py +0 -91
- simple_module_background_tasks-0.0.32/tests/conftest.py +0 -51
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/.gitignore +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/LICENSE +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/README.md +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/__init__.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/_signal_support.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/celery_app.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/__init__.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/events.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/__init__.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/log_context.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/models.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/components/TasksEmpty.tsx +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/components/WorkerHealthBanner.tsx +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/worker-health.ts +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/py.typed +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/signals.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/sync_db.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/tasks.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/package.json +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_beat_schedule.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_index_worker_presence.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_list_item_payload.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_page_clamp.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_log_context.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_signals.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_sweep_stuck.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_sync_db.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_worker_inspector.py +0 -0
- {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_workers_endpoints.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: simple_module_background_tasks
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.34
|
|
4
4
|
Summary: Celery + Redis task queue with admin UI for monitoring and retrying failed/stuck tasks
|
|
5
5
|
Project-URL: Homepage, https://github.com/antosubash/simple_module_python
|
|
6
6
|
Project-URL: Repository, https://github.com/antosubash/simple_module_python
|
|
@@ -23,10 +23,10 @@ Classifier: Typing :: Typed
|
|
|
23
23
|
Requires-Python: >=3.12
|
|
24
24
|
Requires-Dist: celery[redis]>=5.4
|
|
25
25
|
Requires-Dist: redis>=5
|
|
26
|
-
Requires-Dist: simple-module-core==0.0.
|
|
27
|
-
Requires-Dist: simple-module-db==0.0.
|
|
28
|
-
Requires-Dist: simple-module-hosting==0.0.
|
|
29
|
-
Requires-Dist: simple-module-settings==0.0.
|
|
26
|
+
Requires-Dist: simple-module-core==0.0.34
|
|
27
|
+
Requires-Dist: simple-module-db==0.0.34
|
|
28
|
+
Requires-Dist: simple-module-hosting==0.0.34
|
|
29
|
+
Requires-Dist: simple-module-settings==0.0.34
|
|
30
30
|
Description-Content-Type: text/markdown
|
|
31
31
|
|
|
32
32
|
# simple_module_background_tasks
|
|
@@ -25,8 +25,14 @@ _MODULE_SETTINGS = "Settings"
|
|
|
25
25
|
# ── Env / settings ──────────────────────────────────────────────
|
|
26
26
|
ENV_PREFIX = "SM_BG_TASKS_"
|
|
27
27
|
|
|
28
|
+
# The single Redis URL an operator sets. Seeds both the broker and the result
|
|
29
|
+
# backend: Celery namespaces result keys as ``celery-task-meta-*``, so sharing
|
|
30
|
+
# one database is safe and is what upstream's quickstart does. The per-field
|
|
31
|
+
# DB-backed overrides still allow splitting them.
|
|
32
|
+
ENV_REDIS_URL = "SM_REDIS_URL"
|
|
33
|
+
|
|
28
34
|
# ── Permissions ─────────────────────────────────────────────────
|
|
29
|
-
PERM_GROUP = "Background
|
|
35
|
+
PERM_GROUP = "Background tasks"
|
|
30
36
|
PERM_VIEW = "background_tasks.view"
|
|
31
37
|
PERM_MANAGE = "background_tasks.manage"
|
|
32
38
|
|
|
@@ -36,7 +42,7 @@ VIEW_PREFIX = "/admin/background-tasks"
|
|
|
36
42
|
ADMIN_ROUTER_PREFIX = "/admin"
|
|
37
43
|
|
|
38
44
|
# ── Menu ────────────────────────────────────────────────────────
|
|
39
|
-
MENU_LABEL = "Background
|
|
45
|
+
MENU_LABEL = "Background tasks"
|
|
40
46
|
MENU_ICON = "activity"
|
|
41
47
|
MENU_ORDER = 120
|
|
42
48
|
|
|
@@ -59,10 +65,19 @@ class TaskStatus(StrEnum):
|
|
|
59
65
|
|
|
60
66
|
|
|
61
67
|
RETRYABLE_STATUSES: frozenset[TaskStatus] = frozenset({TaskStatus.FAILED, TaskStatus.STUCK})
|
|
68
|
+
|
|
62
69
|
TERMINAL_STATUSES: frozenset[TaskStatus] = frozenset(
|
|
63
70
|
{TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.STUCK, TaskStatus.REVOKED}
|
|
64
71
|
)
|
|
65
72
|
|
|
73
|
+
# How many executions one "Retry all failed" press may re-enqueue.
|
|
74
|
+
#
|
|
75
|
+
# The sweep publishes to the broker once per row and writes a row per publish,
|
|
76
|
+
# so an unbounded backlog turns one button into a multi-minute request holding
|
|
77
|
+
# a DB transaction open. Capping it makes the cost of a press knowable; the
|
|
78
|
+
# response says how many are left so the operator can press again.
|
|
79
|
+
RETRY_ALL_BATCH = 500
|
|
80
|
+
|
|
66
81
|
# ── Defaults ────────────────────────────────────────────────────
|
|
67
82
|
DEFAULT_QUEUE = "default"
|
|
68
83
|
DEFAULT_BROKER_URL = "redis://localhost:6379/0"
|
|
@@ -72,6 +72,23 @@ class TaskExecutionListResponse(SQLModel):
|
|
|
72
72
|
per_page: int
|
|
73
73
|
status: TaskStatus | None = None
|
|
74
74
|
task_name: str | None = None
|
|
75
|
+
queue: str | None = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class RetryFailedResult(SQLModel):
|
|
79
|
+
"""What a bulk retry actually did.
|
|
80
|
+
|
|
81
|
+
The caller cannot compute either number: the endpoint decides which rows
|
|
82
|
+
count as retryable and caps how many it will take in one pass, so both have
|
|
83
|
+
to come back from the action rather than be inferred from what the screen
|
|
84
|
+
was showing.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
queued: int
|
|
88
|
+
# Eligible rows the batch cap left behind. Non-zero means "press it again";
|
|
89
|
+
# the UI says so rather than leaving the operator to wonder why the backlog
|
|
90
|
+
# only half moved.
|
|
91
|
+
remaining: int = 0
|
|
75
92
|
|
|
76
93
|
|
|
77
94
|
class WorkerInfo(SQLModel):
|
|
@@ -84,6 +101,10 @@ class WorkerInfo(SQLModel):
|
|
|
84
101
|
pool_size: int | None = None
|
|
85
102
|
total_processed: int | None = None
|
|
86
103
|
software: str | None = None
|
|
104
|
+
# Seconds since this worker process started, as it reports them. ``None``
|
|
105
|
+
# for a worker that never answered ``stats()`` — an offline worker has no
|
|
106
|
+
# uptime to claim, and zero would read as "just restarted".
|
|
107
|
+
uptime_seconds: float | None = None
|
|
87
108
|
|
|
88
109
|
|
|
89
110
|
class WorkerSnapshot(SQLModel):
|
|
@@ -4,15 +4,15 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from fastapi import Depends, Request
|
|
6
6
|
from simple_module_core.events import EventBus
|
|
7
|
+
from simple_module_db import RequestSession
|
|
7
8
|
from simple_module_db.deps import get_db
|
|
8
|
-
from sqlalchemy.ext.asyncio import AsyncSession
|
|
9
9
|
|
|
10
10
|
from background_tasks.service import BackgroundTaskService
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
async def get_background_task_service(
|
|
14
14
|
request: Request,
|
|
15
|
-
db:
|
|
15
|
+
db: RequestSession = Depends(get_db),
|
|
16
16
|
) -> BackgroundTaskService:
|
|
17
17
|
services = request.app.state.background_tasks
|
|
18
18
|
bus: EventBus = request.app.state.sm.event_bus
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import asyncio
|
|
6
5
|
import uuid
|
|
7
6
|
|
|
8
7
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
@@ -16,13 +15,14 @@ from background_tasks.constants import (
|
|
|
16
15
|
TaskStatus,
|
|
17
16
|
)
|
|
18
17
|
from background_tasks.contracts.schemas import (
|
|
18
|
+
RetryFailedResult,
|
|
19
19
|
TaskExecutionDetail,
|
|
20
20
|
TaskExecutionListResponse,
|
|
21
21
|
WorkerSnapshot,
|
|
22
22
|
)
|
|
23
23
|
from background_tasks.deps import get_background_task_service
|
|
24
24
|
from background_tasks.service import BackgroundTaskService
|
|
25
|
-
from background_tasks.
|
|
25
|
+
from background_tasks.workers_state import poll_workers
|
|
26
26
|
|
|
27
27
|
router = APIRouter(
|
|
28
28
|
prefix=ADMIN_ROUTER_PREFIX,
|
|
@@ -34,12 +34,24 @@ router = APIRouter(
|
|
|
34
34
|
@router.get("/executions", response_model=TaskExecutionListResponse)
|
|
35
35
|
async def list_executions(
|
|
36
36
|
status: TaskStatus | None = Query(default=None),
|
|
37
|
-
|
|
37
|
+
q: str | None = Query(default=None),
|
|
38
|
+
task_name: str | None = Query(default=None, deprecated=True),
|
|
39
|
+
queue: str | None = Query(default=None),
|
|
38
40
|
page: int = Query(default=1, ge=1),
|
|
39
41
|
per_page: int = Query(default=20, ge=1, le=200),
|
|
40
42
|
service: BackgroundTaskService = Depends(get_background_task_service),
|
|
41
43
|
) -> TaskExecutionListResponse:
|
|
42
|
-
|
|
44
|
+
"""List executions, filtered exactly as the page filters them.
|
|
45
|
+
|
|
46
|
+
The task-name search is ``q`` — the same name the page and the sweep use,
|
|
47
|
+
so an operator can move a URL between the three and keep their filter. This
|
|
48
|
+
endpoint originally spelled it ``task_name``, which meant a link copied off
|
|
49
|
+
the screen silently listed *everything*; that spelling is still accepted so
|
|
50
|
+
existing API callers keep working, and ``q`` wins when both are given.
|
|
51
|
+
"""
|
|
52
|
+
return await service.list(
|
|
53
|
+
status=status, task_name=q or task_name, queue=queue, page=page, per_page=per_page
|
|
54
|
+
)
|
|
43
55
|
|
|
44
56
|
|
|
45
57
|
@router.get("/executions/{execution_id}", response_model=TaskExecutionDetail)
|
|
@@ -53,6 +65,26 @@ async def get_execution(
|
|
|
53
65
|
return detail
|
|
54
66
|
|
|
55
67
|
|
|
68
|
+
@router.post(
|
|
69
|
+
"/executions/retry-failed",
|
|
70
|
+
response_model=RetryFailedResult,
|
|
71
|
+
dependencies=[Depends(RequiresPermission(PERM_MANAGE))],
|
|
72
|
+
)
|
|
73
|
+
async def retry_failed_executions(
|
|
74
|
+
status: TaskStatus | None = Query(default=None),
|
|
75
|
+
task_name: str | None = Query(default=None, alias="q"),
|
|
76
|
+
queue: str | None = Query(default=None),
|
|
77
|
+
service: BackgroundTaskService = Depends(get_background_task_service),
|
|
78
|
+
) -> RetryFailedResult:
|
|
79
|
+
"""Re-enqueue every failed or stuck execution the current filter can see.
|
|
80
|
+
|
|
81
|
+
Takes the same three filters as the listing — status, search, queue — so
|
|
82
|
+
the sweep covers exactly the rows the operator is looking at and nothing
|
|
83
|
+
else. Capped per call; the response says how many eligible rows are left.
|
|
84
|
+
"""
|
|
85
|
+
return await service.retry_failed(status=status, task_name=task_name, queue=queue)
|
|
86
|
+
|
|
87
|
+
|
|
56
88
|
@router.post(
|
|
57
89
|
"/executions/{execution_id}/retry",
|
|
58
90
|
response_model=TaskExecutionDetail,
|
|
@@ -67,7 +99,10 @@ async def retry_execution(
|
|
|
67
99
|
|
|
68
100
|
@router.get("/workers", response_model=WorkerSnapshot)
|
|
69
101
|
async def get_workers(request: Request) -> WorkerSnapshot:
|
|
70
|
-
"""Live snapshot of every Celery worker reachable via the broker.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
102
|
+
"""Live snapshot of every Celery worker reachable via the broker.
|
|
103
|
+
|
|
104
|
+
Goes through ``poll_workers`` so the Refresh button also refreshes the
|
|
105
|
+
snapshot other screens read, rather than leaving them on a stale one this
|
|
106
|
+
request has already superseded.
|
|
107
|
+
"""
|
|
108
|
+
return await poll_workers(request)
|
|
@@ -16,12 +16,16 @@ from background_tasks.constants import (
|
|
|
16
16
|
)
|
|
17
17
|
from background_tasks.deps import get_background_task_service
|
|
18
18
|
from background_tasks.service import BackgroundTaskService
|
|
19
|
-
from background_tasks.worker_inspector import
|
|
19
|
+
from background_tasks.worker_inspector import redact_broker_url
|
|
20
|
+
from background_tasks.workers_state import poll_workers
|
|
20
21
|
|
|
21
22
|
router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))])
|
|
22
23
|
|
|
23
24
|
PER_PAGE = 20
|
|
24
25
|
|
|
26
|
+
# Window the "Succeeded 24h" tile counts over.
|
|
27
|
+
SUCCESS_WINDOW_HOURS = 24
|
|
28
|
+
|
|
25
29
|
|
|
26
30
|
async def _worker_presence(request: Request) -> dict[str, object]:
|
|
27
31
|
"""Whether anything is actually able to run a task right now.
|
|
@@ -35,8 +39,7 @@ async def _worker_presence(request: Request) -> dict[str, object]:
|
|
|
35
39
|
unfiltered list is genuinely empty — the one moment the answer changes what
|
|
36
40
|
the screen should say. The normal path never pays for it.
|
|
37
41
|
"""
|
|
38
|
-
|
|
39
|
-
snapshot = await asyncio.to_thread(WorkerInspector(celery).snapshot)
|
|
42
|
+
snapshot = await poll_workers(request)
|
|
40
43
|
return {
|
|
41
44
|
"broker_reachable": snapshot.broker_reachable,
|
|
42
45
|
"worker_count": sum(1 for w in snapshot.workers if w.online),
|
|
@@ -49,35 +52,51 @@ async def index(
|
|
|
49
52
|
request: Request,
|
|
50
53
|
status: TaskStatus | None = Query(default=None),
|
|
51
54
|
task_name: str = Query(default="", alias="q"),
|
|
55
|
+
queue: str = Query(default=""),
|
|
52
56
|
page: int = Query(default=1),
|
|
53
57
|
service: BackgroundTaskService = Depends(get_background_task_service),
|
|
54
58
|
) -> InertiaResponse:
|
|
55
59
|
response = await service.list(
|
|
56
60
|
status=status,
|
|
57
61
|
task_name=task_name or None,
|
|
62
|
+
queue=queue or None,
|
|
58
63
|
page=page,
|
|
59
64
|
per_page=PER_PAGE,
|
|
60
65
|
)
|
|
61
66
|
# A filtered-empty list says nothing about the fleet, so don't pay to poll
|
|
62
67
|
# it: the screen already knows to blame the filter.
|
|
63
|
-
unfiltered_and_empty = response.total == 0 and not task_name and status is None
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
68
|
+
unfiltered_and_empty = response.total == 0 and not task_name and not queue and status is None
|
|
69
|
+
|
|
70
|
+
async def strip_data() -> tuple[dict[str, int], list[str]]:
|
|
71
|
+
"""Everything above the table. Serial: one session, one connection."""
|
|
72
|
+
counts = await service.status_counts(task_name=task_name or None, queue=queue or None)
|
|
73
|
+
# The strip's third tile is a windowed throughput reading rather than a
|
|
74
|
+
# status total, so it needs its own count — see `success_count_since`.
|
|
75
|
+
counts["success_24h"] = await service.success_count_since(
|
|
76
|
+
hours=SUCCESS_WINDOW_HOURS,
|
|
77
|
+
task_name=task_name or None,
|
|
78
|
+
queue=queue or None,
|
|
72
79
|
)
|
|
80
|
+
# Deliberately unfiltered: the dropdown is how an operator leaves the
|
|
81
|
+
# queue they are in, so it keeps offering the ones they are not in.
|
|
82
|
+
return counts, await service.queues()
|
|
83
|
+
|
|
84
|
+
# `_worker_presence` polls the broker over a thread, not the DB session the
|
|
85
|
+
# counts use — the two don't share state, so run them concurrently instead
|
|
86
|
+
# of paying the ~1s inspect timeout serially on top of the queries.
|
|
87
|
+
if unfiltered_and_empty:
|
|
88
|
+
(counts, queues), presence = await asyncio.gather(strip_data(), _worker_presence(request))
|
|
73
89
|
else:
|
|
74
|
-
counts = await
|
|
75
|
-
presence = None
|
|
90
|
+
(counts, queues), presence = await strip_data(), None
|
|
76
91
|
return await inertia.render(
|
|
77
92
|
"BackgroundTasks/Index",
|
|
78
93
|
{
|
|
79
94
|
"executions": [i.model_dump(mode="json") for i in response.items],
|
|
80
|
-
"status_counts": {
|
|
95
|
+
"status_counts": {
|
|
96
|
+
**{s.value: counts.get(s.value, 0) for s in TaskStatus},
|
|
97
|
+
"success_24h": counts["success_24h"],
|
|
98
|
+
},
|
|
99
|
+
"queues": queues,
|
|
81
100
|
"worker_presence": presence,
|
|
82
101
|
"pagination": {
|
|
83
102
|
"page": response.page,
|
|
@@ -87,18 +106,35 @@ async def index(
|
|
|
87
106
|
"filters": {
|
|
88
107
|
"status": status.value if status else "",
|
|
89
108
|
"task_name": task_name,
|
|
109
|
+
"queue": queue,
|
|
90
110
|
},
|
|
91
111
|
},
|
|
92
112
|
)
|
|
93
113
|
|
|
94
114
|
|
|
95
115
|
@router.get("/workers", response_model=None)
|
|
96
|
-
async def workers(
|
|
97
|
-
|
|
98
|
-
|
|
116
|
+
async def workers(
|
|
117
|
+
inertia: InertiaDep,
|
|
118
|
+
request: Request,
|
|
119
|
+
service: BackgroundTaskService = Depends(get_background_task_service),
|
|
120
|
+
) -> InertiaResponse:
|
|
121
|
+
settings = request.app.state.background_tasks.settings
|
|
122
|
+
snapshot = await poll_workers(request)
|
|
123
|
+
# The start command the empty state offers has to name the queues this
|
|
124
|
+
# install actually uses, or it starts a worker that consumes nothing. The
|
|
125
|
+
# default queue leads because it is the one that always exists, even before
|
|
126
|
+
# a single task has run.
|
|
127
|
+
used_queues = await service.queues()
|
|
128
|
+
queues = [settings.task_default_queue] + [
|
|
129
|
+
q for q in used_queues if q != settings.task_default_queue
|
|
130
|
+
]
|
|
99
131
|
return await inertia.render(
|
|
100
132
|
"BackgroundTasks/Workers",
|
|
101
|
-
{
|
|
133
|
+
{
|
|
134
|
+
"snapshot": snapshot.model_dump(mode="json"),
|
|
135
|
+
"broker_url_redacted": redact_broker_url(settings.broker_url),
|
|
136
|
+
"queues": queues,
|
|
137
|
+
},
|
|
102
138
|
)
|
|
103
139
|
|
|
104
140
|
|
|
@@ -106,6 +142,7 @@ async def workers(inertia: InertiaDep, request: Request) -> InertiaResponse:
|
|
|
106
142
|
async def detail(
|
|
107
143
|
execution_id: str,
|
|
108
144
|
inertia: InertiaDep,
|
|
145
|
+
request: Request,
|
|
109
146
|
service: BackgroundTaskService = Depends(get_background_task_service),
|
|
110
147
|
) -> InertiaResponse:
|
|
111
148
|
try:
|
|
@@ -117,5 +154,10 @@ async def detail(
|
|
|
117
154
|
raise HTTPException(status_code=404)
|
|
118
155
|
return await inertia.render(
|
|
119
156
|
"BackgroundTasks/Detail",
|
|
120
|
-
{
|
|
157
|
+
{
|
|
158
|
+
"execution": row.model_dump(mode="json"),
|
|
159
|
+
# The subline reads "attempt {n} of {max}"; without the ceiling the
|
|
160
|
+
# attempt count cannot say whether another one is coming.
|
|
161
|
+
"max_retries": request.app.state.background_tasks.settings.max_retries,
|
|
162
|
+
},
|
|
121
163
|
)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""How the executions screen's filters become SQL.
|
|
2
|
+
|
|
3
|
+
Shared by the listing, the strip counts and the bulk retry. They live out here
|
|
4
|
+
rather than on the service because a bulk action scoped differently from the
|
|
5
|
+
table it sits above is how an operator ends up re-enqueueing rows they never
|
|
6
|
+
saw — one definition means they cannot drift apart.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from simple_module_db import LIKE_ESCAPE_CHAR, like_contains_pattern
|
|
12
|
+
from sqlalchemy import ColumnElement, select
|
|
13
|
+
|
|
14
|
+
from background_tasks.constants import RETRYABLE_STATUSES, TaskStatus
|
|
15
|
+
from background_tasks.models import TaskExecution
|
|
16
|
+
|
|
17
|
+
Conditions = list[ColumnElement[bool]]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def execution_filters(
|
|
21
|
+
*,
|
|
22
|
+
status: TaskStatus | None = None,
|
|
23
|
+
task_name: str | None = None,
|
|
24
|
+
queue: str | None = None,
|
|
25
|
+
) -> Conditions:
|
|
26
|
+
"""The three axes the executions screen filters on."""
|
|
27
|
+
conditions: Conditions = []
|
|
28
|
+
if status is not None:
|
|
29
|
+
conditions.append(TaskExecution.status == status)
|
|
30
|
+
if task_name:
|
|
31
|
+
conditions.append(
|
|
32
|
+
TaskExecution.task_name.ilike(like_contains_pattern(task_name), escape=LIKE_ESCAPE_CHAR)
|
|
33
|
+
)
|
|
34
|
+
if queue:
|
|
35
|
+
conditions.append(TaskExecution.queue == queue)
|
|
36
|
+
return conditions
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def bulk_retry_conditions(
|
|
40
|
+
*,
|
|
41
|
+
status: TaskStatus | None = None,
|
|
42
|
+
task_name: str | None = None,
|
|
43
|
+
queue: str | None = None,
|
|
44
|
+
) -> Conditions | None:
|
|
45
|
+
"""What the bulk sweep may touch, or ``None`` when it may touch nothing.
|
|
46
|
+
|
|
47
|
+
Two rules on top of the screen's own filters.
|
|
48
|
+
|
|
49
|
+
A status that is not retryable narrows the set to empty rather than
|
|
50
|
+
widening it: pressing the button while looking at ``running`` queues
|
|
51
|
+
nothing, because the button can never reach past what is on screen.
|
|
52
|
+
|
|
53
|
+
A row that already has a child has already been re-queued, and sweeping it
|
|
54
|
+
again would multiply the backlog on every press — two clicks on a wedged
|
|
55
|
+
queue of 40 would enqueue 80 tasks. Excluding it makes the button
|
|
56
|
+
idempotent in the way an operator expects. Retrying such a row
|
|
57
|
+
*deliberately* is still possible through its own dialog, which says how
|
|
58
|
+
many attempts it has had and makes the person decide.
|
|
59
|
+
"""
|
|
60
|
+
wanted = RETRYABLE_STATUSES if status is None else RETRYABLE_STATUSES & {status}
|
|
61
|
+
if not wanted:
|
|
62
|
+
return None
|
|
63
|
+
child = TaskExecution.__table__.alias("retry_child")
|
|
64
|
+
already_retried = select(child.c.id).where(child.c.retried_from_id == TaskExecution.id).exists()
|
|
65
|
+
return [
|
|
66
|
+
TaskExecution.status.in_(sorted(wanted)),
|
|
67
|
+
*execution_filters(task_name=task_name, queue=queue),
|
|
68
|
+
~already_retried,
|
|
69
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Health check for the Celery broker and result backend.
|
|
2
|
+
|
|
3
|
+
Doubles as the "Test connection" action on the module-settings screen and as
|
|
4
|
+
the connection step of the first-run setup wizard. Without it a wrong Redis
|
|
5
|
+
URL stays invisible until the first task is enqueued and silently never runs —
|
|
6
|
+
the queue accepts the message, no worker is listening on that database, and
|
|
7
|
+
nothing surfaces an error.
|
|
8
|
+
|
|
9
|
+
The detail string carries the underlying exception verbatim, because
|
|
10
|
+
"connection refused", "authentication required" and "wrong database number"
|
|
11
|
+
need three different fixes and a bare "unhealthy" tells an operator none of
|
|
12
|
+
them.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
|
|
19
|
+
from fastapi import FastAPI
|
|
20
|
+
from simple_module_core.health import HealthCheckResult, HealthStatus
|
|
21
|
+
|
|
22
|
+
CHECK_REDIS = "background_tasks.redis"
|
|
23
|
+
|
|
24
|
+
# kombu's connection is blocking, so it runs in a worker thread. One retry
|
|
25
|
+
# would double the probe's worst-case latency for no diagnostic gain — the
|
|
26
|
+
# point is to report the failure, not to ride it out.
|
|
27
|
+
_CONNECT_TIMEOUT_SECONDS = 3.0
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _ping(celery) -> None:
|
|
31
|
+
"""Open and close a broker connection. Raises on any failure."""
|
|
32
|
+
connection = celery.connection()
|
|
33
|
+
try:
|
|
34
|
+
connection.ensure_connection(max_retries=0, timeout=_CONNECT_TIMEOUT_SECONDS)
|
|
35
|
+
finally:
|
|
36
|
+
connection.release()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def build_redis_check(app: FastAPI):
|
|
40
|
+
"""Return an async check closing over *app* so it re-reads live settings."""
|
|
41
|
+
|
|
42
|
+
async def check() -> HealthCheckResult:
|
|
43
|
+
services = getattr(app.state, "background_tasks", None)
|
|
44
|
+
celery = getattr(services, "celery", None)
|
|
45
|
+
if celery is None:
|
|
46
|
+
return HealthCheckResult(
|
|
47
|
+
status=HealthStatus.UNHEALTHY, detail="Celery app not initialised"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
await asyncio.to_thread(_ping, celery)
|
|
52
|
+
except Exception as exc:
|
|
53
|
+
return HealthCheckResult(status=HealthStatus.UNHEALTHY, detail=str(exc))
|
|
54
|
+
|
|
55
|
+
broker = getattr(services.settings, "broker_url", "")
|
|
56
|
+
return HealthCheckResult(status=HealthStatus.HEALTHY, detail=f"Broker reachable ({broker})")
|
|
57
|
+
|
|
58
|
+
return check
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
{
|
|
2
|
+
"index": {
|
|
3
|
+
"title": "Background tasks",
|
|
4
|
+
"description": "Monitor executions and retry failed or stuck jobs",
|
|
5
|
+
"search_placeholder": "Search by task name…",
|
|
6
|
+
"workers_button": "Workers",
|
|
7
|
+
"retry_all_button": "Retry all failed",
|
|
8
|
+
"previous": "Previous",
|
|
9
|
+
"next": "Next",
|
|
10
|
+
"showing": "Showing {from}–{to} of {total}"
|
|
11
|
+
},
|
|
12
|
+
"strip": {
|
|
13
|
+
"queued": "Queued",
|
|
14
|
+
"running": "Running",
|
|
15
|
+
"succeeded": "Succeeded",
|
|
16
|
+
"failed": "Failed",
|
|
17
|
+
"stuck": "Stuck",
|
|
18
|
+
"window_all": "All time",
|
|
19
|
+
"window_24h": "Last 24h"
|
|
20
|
+
},
|
|
21
|
+
"filters": {
|
|
22
|
+
"status_label": "Filter by status",
|
|
23
|
+
"all": "All",
|
|
24
|
+
"failed": "Failed",
|
|
25
|
+
"running": "Running",
|
|
26
|
+
"stuck": "Stuck",
|
|
27
|
+
"queue_label": "Queue:",
|
|
28
|
+
"queue_all": "all",
|
|
29
|
+
"queue_aria": "Filter by queue"
|
|
30
|
+
},
|
|
31
|
+
"status": {
|
|
32
|
+
"queued": "queued",
|
|
33
|
+
"running": "running",
|
|
34
|
+
"success": "success",
|
|
35
|
+
"failed": "failed",
|
|
36
|
+
"stuck": "stuck",
|
|
37
|
+
"revoked": "revoked",
|
|
38
|
+
"retrying": "retrying"
|
|
39
|
+
},
|
|
40
|
+
"table": {
|
|
41
|
+
"task": "Task",
|
|
42
|
+
"status": "Status",
|
|
43
|
+
"queue": "Queue",
|
|
44
|
+
"queued_at": "Queued",
|
|
45
|
+
"duration": "Duration",
|
|
46
|
+
"actions": "Actions",
|
|
47
|
+
"retry": "Retry"
|
|
48
|
+
},
|
|
49
|
+
"detail": {
|
|
50
|
+
"head_title": "Task detail",
|
|
51
|
+
"description": "execution {id} · attempt {attempt} of {max}",
|
|
52
|
+
"attempt": "attempt {attempt} of {max}",
|
|
53
|
+
"back_button": "Back to executions",
|
|
54
|
+
"retry_button": "Retry task",
|
|
55
|
+
"meta": "Details",
|
|
56
|
+
"queue": "Queue",
|
|
57
|
+
"worker": "Worker",
|
|
58
|
+
"celery_id": "Celery id",
|
|
59
|
+
"queued_at": "Queued at",
|
|
60
|
+
"started_at": "Started at",
|
|
61
|
+
"finished_at": "Finished at",
|
|
62
|
+
"duration": "Duration",
|
|
63
|
+
"heartbeat": "Heartbeat",
|
|
64
|
+
"retried_from": "Retried from",
|
|
65
|
+
"exception": "Exception",
|
|
66
|
+
"args": "Arguments",
|
|
67
|
+
"kwargs": "Keyword arguments",
|
|
68
|
+
"result": "Result",
|
|
69
|
+
"traceback": "Traceback",
|
|
70
|
+
"copy": "Copy",
|
|
71
|
+
"copied": "Copied",
|
|
72
|
+
"no_traceback": "No traceback recorded."
|
|
73
|
+
},
|
|
74
|
+
"retry_dialog": {
|
|
75
|
+
"title": "Retry {name}?",
|
|
76
|
+
"description": "A new execution is queued with the same arguments.",
|
|
77
|
+
"retried_before_one": "This one has already been retried once.",
|
|
78
|
+
"retried_before_other": "This one has already been retried {count} times.",
|
|
79
|
+
"args_label": "args",
|
|
80
|
+
"kwargs_label": "kwargs",
|
|
81
|
+
"no_args": "This task takes no arguments.",
|
|
82
|
+
"cancel": "Cancel",
|
|
83
|
+
"confirm": "Queue retry"
|
|
84
|
+
},
|
|
85
|
+
"retry_all_dialog": {
|
|
86
|
+
"title": "Retry all failed tasks?",
|
|
87
|
+
"description": "Every failed or stuck execution in the current view is queued again with its original arguments. Rows that already have a retry in flight are skipped.",
|
|
88
|
+
"scope_all_queues": "All queues",
|
|
89
|
+
"scope_queue": "Queue {queue}",
|
|
90
|
+
"scope_all_statuses": "Failed and stuck",
|
|
91
|
+
"cancel": "Cancel",
|
|
92
|
+
"confirm": "Queue retries",
|
|
93
|
+
"scope_search": "Search “{query}”",
|
|
94
|
+
"scope_no_search": "Any task name"
|
|
95
|
+
},
|
|
96
|
+
"toasts": {
|
|
97
|
+
"retried": "Task \"{name}\" re-enqueued",
|
|
98
|
+
"retry_failed": "Failed to retry task",
|
|
99
|
+
"bulk_retried_one": "Queued {count} retry",
|
|
100
|
+
"bulk_retried_other": "Queued {count} retries",
|
|
101
|
+
"bulk_retry_failed": "Failed to retry tasks",
|
|
102
|
+
"bulk_remaining_one": "{count} more remains — run it again.",
|
|
103
|
+
"bulk_remaining_other": "{count} more remain — run it again."
|
|
104
|
+
},
|
|
105
|
+
"worker_health": {
|
|
106
|
+
"broker_unreachable_title": "Broker unreachable",
|
|
107
|
+
"broker_unreachable_detail": "Nothing can be queued or run until the broker is back. The counts below are the last state written to the database.",
|
|
108
|
+
"no_workers_title_one": "{count} task waiting, no worker running",
|
|
109
|
+
"no_workers_title_other": "{count} tasks waiting, no worker running",
|
|
110
|
+
"no_workers_detail": "The broker is up but nothing is consuming the queue, so this backlog will not move on its own.",
|
|
111
|
+
"view_workers": "View workers"
|
|
112
|
+
},
|
|
113
|
+
"tasks_empty": {
|
|
114
|
+
"filtered_title": "No tasks match these filters",
|
|
115
|
+
"filtered_description": "Executions exist outside the current search, status and queue filters.",
|
|
116
|
+
"clear_filters": "Clear filters",
|
|
117
|
+
"broker_unreachable_title": "Can't reach the broker",
|
|
118
|
+
"broker_unreachable_description": "Nothing can be queued or executed until the broker is back. Tasks are not being lost — they are not being accepted.",
|
|
119
|
+
"no_workers_title": "No worker is running",
|
|
120
|
+
"no_workers_description": "The broker is up, but nothing is consuming the queue. Tasks enqueued now will wait rather than run.",
|
|
121
|
+
"healthy_title": "No task has run yet",
|
|
122
|
+
"healthy_description": "Registered tasks appear here the first time they execute."
|
|
123
|
+
},
|
|
124
|
+
"workers": {
|
|
125
|
+
"title": "Workers",
|
|
126
|
+
"description": "Celery workers connected to the broker",
|
|
127
|
+
"back_button": "Executions",
|
|
128
|
+
"last_updated": "Last updated {time}",
|
|
129
|
+
"refresh": "Refresh",
|
|
130
|
+
"online": "Online",
|
|
131
|
+
"offline": "Offline",
|
|
132
|
+
"uptime": "uptime {uptime}",
|
|
133
|
+
"offline_state": "offline",
|
|
134
|
+
"active": "Active",
|
|
135
|
+
"pool": "Pool",
|
|
136
|
+
"processed": "Processed",
|
|
137
|
+
"queues": "Queues",
|
|
138
|
+
"broker_unreachable_title": "Broker unreachable",
|
|
139
|
+
"broker_unreachable_description": "The connection to the broker failed, so nothing can be queued or run. Check this setting and confirm the broker is up.",
|
|
140
|
+
"no_error_reported": "No error message reported.",
|
|
141
|
+
"no_workers_title": "No workers connected",
|
|
142
|
+
"no_workers_description": "The broker is reachable but nothing is consuming the queue. Start a worker locally with:",
|
|
143
|
+
"refresh_failed_status": "Failed to refresh workers (HTTP {status})",
|
|
144
|
+
"refresh_failed": "Failed to refresh workers"
|
|
145
|
+
},
|
|
146
|
+
"nav": {
|
|
147
|
+
"background_tasks": "Background tasks"
|
|
148
|
+
},
|
|
149
|
+
"audit": {
|
|
150
|
+
"task_execution": "Task execution"
|
|
151
|
+
}
|
|
152
|
+
}
|