simple-module-background-tasks 0.0.32__tar.gz → 0.0.34__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/PKG-INFO +5 -5
  2. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/constants.py +17 -2
  3. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/schemas.py +21 -0
  4. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/deps.py +2 -2
  5. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/api_admin.py +43 -8
  6. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/views.py +62 -20
  7. simple_module_background_tasks-0.0.34/background_tasks/filters.py +69 -0
  8. simple_module_background_tasks-0.0.34/background_tasks/health.py +58 -0
  9. simple_module_background_tasks-0.0.34/background_tasks/locales/en.json +152 -0
  10. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/module.py +66 -2
  11. simple_module_background_tasks-0.0.34/background_tasks/pages/Detail.tsx +129 -0
  12. simple_module_background_tasks-0.0.34/background_tasks/pages/Index.tsx +232 -0
  13. simple_module_background_tasks-0.0.34/background_tasks/pages/Workers.tsx +115 -0
  14. simple_module_background_tasks-0.0.34/background_tasks/pages/components/DetailFacts.tsx +118 -0
  15. simple_module_background_tasks-0.0.34/background_tasks/pages/components/ExecutionRow.tsx +68 -0
  16. simple_module_background_tasks-0.0.34/background_tasks/pages/components/ExecutionsTable.tsx +120 -0
  17. simple_module_background_tasks-0.0.34/background_tasks/pages/components/PayloadCards.tsx +50 -0
  18. simple_module_background_tasks-0.0.34/background_tasks/pages/components/RetryAllDialog.tsx +76 -0
  19. simple_module_background_tasks-0.0.34/background_tasks/pages/components/RetryConfirmDialog.tsx +93 -0
  20. simple_module_background_tasks-0.0.34/background_tasks/pages/components/StatusPill.tsx +30 -0
  21. simple_module_background_tasks-0.0.34/background_tasks/pages/components/StatusStrip.tsx +128 -0
  22. simple_module_background_tasks-0.0.34/background_tasks/pages/components/TaskFilters.tsx +93 -0
  23. simple_module_background_tasks-0.0.34/background_tasks/pages/components/TracebackCard.tsx +81 -0
  24. simple_module_background_tasks-0.0.34/background_tasks/pages/components/WorkerCard.tsx +102 -0
  25. simple_module_background_tasks-0.0.34/background_tasks/pages/components/WorkerEmptyStates.tsx +79 -0
  26. simple_module_background_tasks-0.0.34/background_tasks/pages/constants.ts +255 -0
  27. simple_module_background_tasks-0.0.34/background_tasks/pages/retry.ts +94 -0
  28. simple_module_background_tasks-0.0.34/background_tasks/retry_service.py +220 -0
  29. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/service.py +77 -51
  30. simple_module_background_tasks-0.0.34/background_tasks/services.py +37 -0
  31. simple_module_background_tasks-0.0.34/background_tasks/settings.py +170 -0
  32. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/worker_inspector.py +72 -2
  33. simple_module_background_tasks-0.0.34/background_tasks/workers_state.py +31 -0
  34. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/pyproject.toml +5 -5
  35. simple_module_background_tasks-0.0.34/tests/conftest.py +113 -0
  36. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_admin_api.py +29 -0
  37. simple_module_background_tasks-0.0.34/tests/test_background_tasks_settings.py +27 -0
  38. simple_module_background_tasks-0.0.34/tests/test_bg_redis_health.py +121 -0
  39. simple_module_background_tasks-0.0.34/tests/test_bg_redis_url.py +106 -0
  40. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_service.py +19 -12
  41. simple_module_background_tasks-0.0.34/tests/test_bg_settings_env.py +118 -0
  42. simple_module_background_tasks-0.0.34/tests/test_detail_props.py +78 -0
  43. simple_module_background_tasks-0.0.34/tests/test_index_filters.py +186 -0
  44. simple_module_background_tasks-0.0.34/tests/test_retry_failed_bulk.py +150 -0
  45. simple_module_background_tasks-0.0.34/tests/test_retry_failed_guards.py +155 -0
  46. simple_module_background_tasks-0.0.34/tests/test_retry_sweep_ordering.py +200 -0
  47. simple_module_background_tasks-0.0.34/tests/test_workers_uptime.py +241 -0
  48. simple_module_background_tasks-0.0.34/tests-js/task-formatters.test.ts +150 -0
  49. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tsconfig.json +6 -1
  50. simple_module_background_tasks-0.0.32/background_tasks/locales/en.json +0 -76
  51. simple_module_background_tasks-0.0.32/background_tasks/pages/Detail.tsx +0 -188
  52. simple_module_background_tasks-0.0.32/background_tasks/pages/Index.tsx +0 -240
  53. simple_module_background_tasks-0.0.32/background_tasks/pages/Workers.tsx +0 -186
  54. simple_module_background_tasks-0.0.32/background_tasks/pages/components/ExecutionRow.tsx +0 -82
  55. simple_module_background_tasks-0.0.32/background_tasks/pages/components/RetryConfirmDialog.tsx +0 -81
  56. simple_module_background_tasks-0.0.32/background_tasks/pages/components/StatusStrip.tsx +0 -66
  57. simple_module_background_tasks-0.0.32/background_tasks/pages/constants.ts +0 -70
  58. simple_module_background_tasks-0.0.32/background_tasks/pages/retry.ts +0 -45
  59. simple_module_background_tasks-0.0.32/background_tasks/services.py +0 -25
  60. simple_module_background_tasks-0.0.32/background_tasks/settings.py +0 -91
  61. simple_module_background_tasks-0.0.32/tests/conftest.py +0 -51
  62. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/.gitignore +0 -0
  63. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/LICENSE +0 -0
  64. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/README.md +0 -0
  65. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/__init__.py +0 -0
  66. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/_signal_support.py +0 -0
  67. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/celery_app.py +0 -0
  68. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/__init__.py +0 -0
  69. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/contracts/events.py +0 -0
  70. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/endpoints/__init__.py +0 -0
  71. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/log_context.py +0 -0
  72. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/models.py +0 -0
  73. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/components/TasksEmpty.tsx +0 -0
  74. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/components/WorkerHealthBanner.tsx +0 -0
  75. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/pages/worker-health.ts +0 -0
  76. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/py.typed +0 -0
  77. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/signals.py +0 -0
  78. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/sync_db.py +0 -0
  79. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/background_tasks/tasks.py +0 -0
  80. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/package.json +0 -0
  81. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_beat_schedule.py +0 -0
  82. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_index_worker_presence.py +0 -0
  83. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_list_item_payload.py +0 -0
  84. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_bg_page_clamp.py +0 -0
  85. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_log_context.py +0 -0
  86. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_signals.py +0 -0
  87. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_sweep_stuck.py +0 -0
  88. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_sync_db.py +0 -0
  89. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_worker_inspector.py +0 -0
  90. {simple_module_background_tasks-0.0.32 → simple_module_background_tasks-0.0.34}/tests/test_workers_endpoints.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: simple_module_background_tasks
3
- Version: 0.0.32
3
+ Version: 0.0.34
4
4
  Summary: Celery + Redis task queue with admin UI for monitoring and retrying failed/stuck tasks
5
5
  Project-URL: Homepage, https://github.com/antosubash/simple_module_python
6
6
  Project-URL: Repository, https://github.com/antosubash/simple_module_python
@@ -23,10 +23,10 @@ Classifier: Typing :: Typed
23
23
  Requires-Python: >=3.12
24
24
  Requires-Dist: celery[redis]>=5.4
25
25
  Requires-Dist: redis>=5
26
- Requires-Dist: simple-module-core==0.0.32
27
- Requires-Dist: simple-module-db==0.0.32
28
- Requires-Dist: simple-module-hosting==0.0.32
29
- Requires-Dist: simple-module-settings==0.0.32
26
+ Requires-Dist: simple-module-core==0.0.34
27
+ Requires-Dist: simple-module-db==0.0.34
28
+ Requires-Dist: simple-module-hosting==0.0.34
29
+ Requires-Dist: simple-module-settings==0.0.34
30
30
  Description-Content-Type: text/markdown
31
31
 
32
32
  # simple_module_background_tasks
@@ -25,8 +25,14 @@ _MODULE_SETTINGS = "Settings"
25
25
  # ── Env / settings ──────────────────────────────────────────────
26
26
  ENV_PREFIX = "SM_BG_TASKS_"
27
27
 
28
+ # The single Redis URL an operator sets. Seeds both the broker and the result
29
+ # backend: Celery namespaces result keys as ``celery-task-meta-*``, so sharing
30
+ # one database is safe and is what upstream's quickstart does. The per-field
31
+ # DB-backed overrides still allow splitting them.
32
+ ENV_REDIS_URL = "SM_REDIS_URL"
33
+
28
34
  # ── Permissions ─────────────────────────────────────────────────
29
- PERM_GROUP = "Background Tasks"
35
+ PERM_GROUP = "Background tasks"
30
36
  PERM_VIEW = "background_tasks.view"
31
37
  PERM_MANAGE = "background_tasks.manage"
32
38
 
@@ -36,7 +42,7 @@ VIEW_PREFIX = "/admin/background-tasks"
36
42
  ADMIN_ROUTER_PREFIX = "/admin"
37
43
 
38
44
  # ── Menu ────────────────────────────────────────────────────────
39
- MENU_LABEL = "Background Tasks"
45
+ MENU_LABEL = "Background tasks"
40
46
  MENU_ICON = "activity"
41
47
  MENU_ORDER = 120
42
48
 
@@ -59,10 +65,19 @@ class TaskStatus(StrEnum):
59
65
 
60
66
 
61
67
  RETRYABLE_STATUSES: frozenset[TaskStatus] = frozenset({TaskStatus.FAILED, TaskStatus.STUCK})
68
+
62
69
  TERMINAL_STATUSES: frozenset[TaskStatus] = frozenset(
63
70
  {TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.STUCK, TaskStatus.REVOKED}
64
71
  )
65
72
 
73
+ # How many executions one "Retry all failed" press may re-enqueue.
74
+ #
75
+ # The sweep publishes to the broker once per row and writes a row per publish,
76
+ # so an unbounded backlog turns one button into a multi-minute request holding
77
+ # a DB transaction open. Capping it makes the cost of a press knowable; the
78
+ # response says how many are left so the operator can press again.
79
+ RETRY_ALL_BATCH = 500
80
+
66
81
  # ── Defaults ────────────────────────────────────────────────────
67
82
  DEFAULT_QUEUE = "default"
68
83
  DEFAULT_BROKER_URL = "redis://localhost:6379/0"
@@ -72,6 +72,23 @@ class TaskExecutionListResponse(SQLModel):
72
72
  per_page: int
73
73
  status: TaskStatus | None = None
74
74
  task_name: str | None = None
75
+ queue: str | None = None
76
+
77
+
78
+ class RetryFailedResult(SQLModel):
79
+ """What a bulk retry actually did.
80
+
81
+ The caller cannot compute either number: the endpoint decides which rows
82
+ count as retryable and caps how many it will take in one pass, so both have
83
+ to come back from the action rather than be inferred from what the screen
84
+ was showing.
85
+ """
86
+
87
+ queued: int
88
+ # Eligible rows the batch cap left behind. Non-zero means "press it again";
89
+ # the UI says so rather than leaving the operator to wonder why the backlog
90
+ # only half moved.
91
+ remaining: int = 0
75
92
 
76
93
 
77
94
  class WorkerInfo(SQLModel):
@@ -84,6 +101,10 @@ class WorkerInfo(SQLModel):
84
101
  pool_size: int | None = None
85
102
  total_processed: int | None = None
86
103
  software: str | None = None
104
+ # Seconds since this worker process started, as it reports them. ``None``
105
+ # for a worker that never answered ``stats()`` — an offline worker has no
106
+ # uptime to claim, and zero would read as "just restarted".
107
+ uptime_seconds: float | None = None
87
108
 
88
109
 
89
110
  class WorkerSnapshot(SQLModel):
@@ -4,15 +4,15 @@ from __future__ import annotations
4
4
 
5
5
  from fastapi import Depends, Request
6
6
  from simple_module_core.events import EventBus
7
+ from simple_module_db import RequestSession
7
8
  from simple_module_db.deps import get_db
8
- from sqlalchemy.ext.asyncio import AsyncSession
9
9
 
10
10
  from background_tasks.service import BackgroundTaskService
11
11
 
12
12
 
13
13
  async def get_background_task_service(
14
14
  request: Request,
15
- db: AsyncSession = Depends(get_db),
15
+ db: RequestSession = Depends(get_db),
16
16
  ) -> BackgroundTaskService:
17
17
  services = request.app.state.background_tasks
18
18
  bus: EventBus = request.app.state.sm.event_bus
@@ -2,7 +2,6 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- import asyncio
6
5
  import uuid
7
6
 
8
7
  from fastapi import APIRouter, Depends, HTTPException, Query, Request
@@ -16,13 +15,14 @@ from background_tasks.constants import (
16
15
  TaskStatus,
17
16
  )
18
17
  from background_tasks.contracts.schemas import (
18
+ RetryFailedResult,
19
19
  TaskExecutionDetail,
20
20
  TaskExecutionListResponse,
21
21
  WorkerSnapshot,
22
22
  )
23
23
  from background_tasks.deps import get_background_task_service
24
24
  from background_tasks.service import BackgroundTaskService
25
- from background_tasks.worker_inspector import WorkerInspector
25
+ from background_tasks.workers_state import poll_workers
26
26
 
27
27
  router = APIRouter(
28
28
  prefix=ADMIN_ROUTER_PREFIX,
@@ -34,12 +34,24 @@ router = APIRouter(
34
34
  @router.get("/executions", response_model=TaskExecutionListResponse)
35
35
  async def list_executions(
36
36
  status: TaskStatus | None = Query(default=None),
37
- task_name: str | None = Query(default=None),
37
+ q: str | None = Query(default=None),
38
+ task_name: str | None = Query(default=None, deprecated=True),
39
+ queue: str | None = Query(default=None),
38
40
  page: int = Query(default=1, ge=1),
39
41
  per_page: int = Query(default=20, ge=1, le=200),
40
42
  service: BackgroundTaskService = Depends(get_background_task_service),
41
43
  ) -> TaskExecutionListResponse:
42
- return await service.list(status=status, task_name=task_name, page=page, per_page=per_page)
44
+ """List executions, filtered exactly as the page filters them.
45
+
46
+ The task-name search is ``q`` — the same name the page and the sweep use,
47
+ so an operator can move a URL between the three and keep their filter. This
48
+ endpoint originally spelled it ``task_name``, which meant a link copied off
49
+ the screen silently listed *everything*; that spelling is still accepted so
50
+ existing API callers keep working, and ``q`` wins when both are given.
51
+ """
52
+ return await service.list(
53
+ status=status, task_name=q or task_name, queue=queue, page=page, per_page=per_page
54
+ )
43
55
 
44
56
 
45
57
  @router.get("/executions/{execution_id}", response_model=TaskExecutionDetail)
@@ -53,6 +65,26 @@ async def get_execution(
53
65
  return detail
54
66
 
55
67
 
68
+ @router.post(
69
+ "/executions/retry-failed",
70
+ response_model=RetryFailedResult,
71
+ dependencies=[Depends(RequiresPermission(PERM_MANAGE))],
72
+ )
73
+ async def retry_failed_executions(
74
+ status: TaskStatus | None = Query(default=None),
75
+ task_name: str | None = Query(default=None, alias="q"),
76
+ queue: str | None = Query(default=None),
77
+ service: BackgroundTaskService = Depends(get_background_task_service),
78
+ ) -> RetryFailedResult:
79
+ """Re-enqueue every failed or stuck execution the current filter can see.
80
+
81
+ Takes the same three filters as the listing — status, search, queue — so
82
+ the sweep covers exactly the rows the operator is looking at and nothing
83
+ else. Capped per call; the response says how many eligible rows are left.
84
+ """
85
+ return await service.retry_failed(status=status, task_name=task_name, queue=queue)
86
+
87
+
56
88
  @router.post(
57
89
  "/executions/{execution_id}/retry",
58
90
  response_model=TaskExecutionDetail,
@@ -67,7 +99,10 @@ async def retry_execution(
67
99
 
68
100
  @router.get("/workers", response_model=WorkerSnapshot)
69
101
  async def get_workers(request: Request) -> WorkerSnapshot:
70
- """Live snapshot of every Celery worker reachable via the broker."""
71
- celery = request.app.state.background_tasks.celery
72
- inspector = WorkerInspector(celery)
73
- return await asyncio.to_thread(inspector.snapshot)
102
+ """Live snapshot of every Celery worker reachable via the broker.
103
+
104
+ Goes through ``poll_workers`` so the Refresh button also refreshes the
105
+ snapshot other screens read, rather than leaving them on a stale one this
106
+ request has already superseded.
107
+ """
108
+ return await poll_workers(request)
@@ -16,12 +16,16 @@ from background_tasks.constants import (
16
16
  )
17
17
  from background_tasks.deps import get_background_task_service
18
18
  from background_tasks.service import BackgroundTaskService
19
- from background_tasks.worker_inspector import WorkerInspector
19
+ from background_tasks.worker_inspector import redact_broker_url
20
+ from background_tasks.workers_state import poll_workers
20
21
 
21
22
  router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))])
22
23
 
23
24
  PER_PAGE = 20
24
25
 
26
+ # Window the "Succeeded 24h" tile counts over.
27
+ SUCCESS_WINDOW_HOURS = 24
28
+
25
29
 
26
30
  async def _worker_presence(request: Request) -> dict[str, object]:
27
31
  """Whether anything is actually able to run a task right now.
@@ -35,8 +39,7 @@ async def _worker_presence(request: Request) -> dict[str, object]:
35
39
  unfiltered list is genuinely empty — the one moment the answer changes what
36
40
  the screen should say. The normal path never pays for it.
37
41
  """
38
- celery = request.app.state.background_tasks.celery
39
- snapshot = await asyncio.to_thread(WorkerInspector(celery).snapshot)
42
+ snapshot = await poll_workers(request)
40
43
  return {
41
44
  "broker_reachable": snapshot.broker_reachable,
42
45
  "worker_count": sum(1 for w in snapshot.workers if w.online),
@@ -49,35 +52,51 @@ async def index(
49
52
  request: Request,
50
53
  status: TaskStatus | None = Query(default=None),
51
54
  task_name: str = Query(default="", alias="q"),
55
+ queue: str = Query(default=""),
52
56
  page: int = Query(default=1),
53
57
  service: BackgroundTaskService = Depends(get_background_task_service),
54
58
  ) -> InertiaResponse:
55
59
  response = await service.list(
56
60
  status=status,
57
61
  task_name=task_name or None,
62
+ queue=queue or None,
58
63
  page=page,
59
64
  per_page=PER_PAGE,
60
65
  )
61
66
  # A filtered-empty list says nothing about the fleet, so don't pay to poll
62
67
  # it: the screen already knows to blame the filter.
63
- unfiltered_and_empty = response.total == 0 and not task_name and status is None
64
- # `_worker_presence` polls the broker over a thread, not the DB session
65
- # `status_counts` uses — the two don't share state, so run them
66
- # concurrently instead of paying the ~1s inspect timeout serially on top
67
- # of the count query.
68
- if unfiltered_and_empty:
69
- counts, presence = await asyncio.gather(
70
- service.status_counts(task_name=task_name or None),
71
- _worker_presence(request),
68
+ unfiltered_and_empty = response.total == 0 and not task_name and not queue and status is None
69
+
70
+ async def strip_data() -> tuple[dict[str, int], list[str]]:
71
+ """Everything above the table. Serial: one session, one connection."""
72
+ counts = await service.status_counts(task_name=task_name or None, queue=queue or None)
73
+ # The strip's third tile is a windowed throughput reading rather than a
74
+ # status total, so it needs its own count — see `success_count_since`.
75
+ counts["success_24h"] = await service.success_count_since(
76
+ hours=SUCCESS_WINDOW_HOURS,
77
+ task_name=task_name or None,
78
+ queue=queue or None,
72
79
  )
80
+ # Deliberately unfiltered: the dropdown is how an operator leaves the
81
+ # queue they are in, so it keeps offering the ones they are not in.
82
+ return counts, await service.queues()
83
+
84
+ # `_worker_presence` polls the broker over a thread, not the DB session the
85
+ # counts use — the two don't share state, so run them concurrently instead
86
+ # of paying the ~1s inspect timeout serially on top of the queries.
87
+ if unfiltered_and_empty:
88
+ (counts, queues), presence = await asyncio.gather(strip_data(), _worker_presence(request))
73
89
  else:
74
- counts = await service.status_counts(task_name=task_name or None)
75
- presence = None
90
+ (counts, queues), presence = await strip_data(), None
76
91
  return await inertia.render(
77
92
  "BackgroundTasks/Index",
78
93
  {
79
94
  "executions": [i.model_dump(mode="json") for i in response.items],
80
- "status_counts": {s.value: counts.get(s.value, 0) for s in TaskStatus},
95
+ "status_counts": {
96
+ **{s.value: counts.get(s.value, 0) for s in TaskStatus},
97
+ "success_24h": counts["success_24h"],
98
+ },
99
+ "queues": queues,
81
100
  "worker_presence": presence,
82
101
  "pagination": {
83
102
  "page": response.page,
@@ -87,18 +106,35 @@ async def index(
87
106
  "filters": {
88
107
  "status": status.value if status else "",
89
108
  "task_name": task_name,
109
+ "queue": queue,
90
110
  },
91
111
  },
92
112
  )
93
113
 
94
114
 
95
115
  @router.get("/workers", response_model=None)
96
- async def workers(inertia: InertiaDep, request: Request) -> InertiaResponse:
97
- celery = request.app.state.background_tasks.celery
98
- snapshot = await asyncio.to_thread(WorkerInspector(celery).snapshot)
116
+ async def workers(
117
+ inertia: InertiaDep,
118
+ request: Request,
119
+ service: BackgroundTaskService = Depends(get_background_task_service),
120
+ ) -> InertiaResponse:
121
+ settings = request.app.state.background_tasks.settings
122
+ snapshot = await poll_workers(request)
123
+ # The start command the empty state offers has to name the queues this
124
+ # install actually uses, or it starts a worker that consumes nothing. The
125
+ # default queue leads because it is the one that always exists, even before
126
+ # a single task has run.
127
+ used_queues = await service.queues()
128
+ queues = [settings.task_default_queue] + [
129
+ q for q in used_queues if q != settings.task_default_queue
130
+ ]
99
131
  return await inertia.render(
100
132
  "BackgroundTasks/Workers",
101
- {"snapshot": snapshot.model_dump(mode="json")},
133
+ {
134
+ "snapshot": snapshot.model_dump(mode="json"),
135
+ "broker_url_redacted": redact_broker_url(settings.broker_url),
136
+ "queues": queues,
137
+ },
102
138
  )
103
139
 
104
140
 
@@ -106,6 +142,7 @@ async def workers(inertia: InertiaDep, request: Request) -> InertiaResponse:
106
142
  async def detail(
107
143
  execution_id: str,
108
144
  inertia: InertiaDep,
145
+ request: Request,
109
146
  service: BackgroundTaskService = Depends(get_background_task_service),
110
147
  ) -> InertiaResponse:
111
148
  try:
@@ -117,5 +154,10 @@ async def detail(
117
154
  raise HTTPException(status_code=404)
118
155
  return await inertia.render(
119
156
  "BackgroundTasks/Detail",
120
- {"execution": row.model_dump(mode="json")},
157
+ {
158
+ "execution": row.model_dump(mode="json"),
159
+ # The subline reads "attempt {n} of {max}"; without the ceiling the
160
+ # attempt count cannot say whether another one is coming.
161
+ "max_retries": request.app.state.background_tasks.settings.max_retries,
162
+ },
121
163
  )
@@ -0,0 +1,69 @@
1
+ """How the executions screen's filters become SQL.
2
+
3
+ Shared by the listing, the strip counts and the bulk retry. They live out here
4
+ rather than on the service because a bulk action scoped differently from the
5
+ table it sits above is how an operator ends up re-enqueueing rows they never
6
+ saw — one definition means they cannot drift apart.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from simple_module_db import LIKE_ESCAPE_CHAR, like_contains_pattern
12
+ from sqlalchemy import ColumnElement, select
13
+
14
+ from background_tasks.constants import RETRYABLE_STATUSES, TaskStatus
15
+ from background_tasks.models import TaskExecution
16
+
17
+ Conditions = list[ColumnElement[bool]]
18
+
19
+
20
+ def execution_filters(
21
+ *,
22
+ status: TaskStatus | None = None,
23
+ task_name: str | None = None,
24
+ queue: str | None = None,
25
+ ) -> Conditions:
26
+ """The three axes the executions screen filters on."""
27
+ conditions: Conditions = []
28
+ if status is not None:
29
+ conditions.append(TaskExecution.status == status)
30
+ if task_name:
31
+ conditions.append(
32
+ TaskExecution.task_name.ilike(like_contains_pattern(task_name), escape=LIKE_ESCAPE_CHAR)
33
+ )
34
+ if queue:
35
+ conditions.append(TaskExecution.queue == queue)
36
+ return conditions
37
+
38
+
39
+ def bulk_retry_conditions(
40
+ *,
41
+ status: TaskStatus | None = None,
42
+ task_name: str | None = None,
43
+ queue: str | None = None,
44
+ ) -> Conditions | None:
45
+ """What the bulk sweep may touch, or ``None`` when it may touch nothing.
46
+
47
+ Two rules on top of the screen's own filters.
48
+
49
+ A status that is not retryable narrows the set to empty rather than
50
+ widening it: pressing the button while looking at ``running`` queues
51
+ nothing, because the button can never reach past what is on screen.
52
+
53
+ A row that already has a child has already been re-queued, and sweeping it
54
+ again would multiply the backlog on every press — two clicks on a wedged
55
+ queue of 40 would enqueue 80 tasks. Excluding it makes the button
56
+ idempotent in the way an operator expects. Retrying such a row
57
+ *deliberately* is still possible through its own dialog, which says how
58
+ many attempts it has had and makes the person decide.
59
+ """
60
+ wanted = RETRYABLE_STATUSES if status is None else RETRYABLE_STATUSES & {status}
61
+ if not wanted:
62
+ return None
63
+ child = TaskExecution.__table__.alias("retry_child")
64
+ already_retried = select(child.c.id).where(child.c.retried_from_id == TaskExecution.id).exists()
65
+ return [
66
+ TaskExecution.status.in_(sorted(wanted)),
67
+ *execution_filters(task_name=task_name, queue=queue),
68
+ ~already_retried,
69
+ ]
@@ -0,0 +1,58 @@
1
+ """Health check for the Celery broker and result backend.
2
+
3
+ Doubles as the "Test connection" action on the module-settings screen and as
4
+ the connection step of the first-run setup wizard. Without it a wrong Redis
5
+ URL stays invisible until the first task is enqueued and silently never runs —
6
+ the queue accepts the message, no worker is listening on that database, and
7
+ nothing surfaces an error.
8
+
9
+ The detail string carries the underlying exception verbatim, because
10
+ "connection refused", "authentication required" and "wrong database number"
11
+ need three different fixes and a bare "unhealthy" tells an operator none of
12
+ them.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import asyncio
18
+
19
+ from fastapi import FastAPI
20
+ from simple_module_core.health import HealthCheckResult, HealthStatus
21
+
22
+ CHECK_REDIS = "background_tasks.redis"
23
+
24
+ # kombu's connection is blocking, so it runs in a worker thread. One retry
25
+ # would double the probe's worst-case latency for no diagnostic gain — the
26
+ # point is to report the failure, not to ride it out.
27
+ _CONNECT_TIMEOUT_SECONDS = 3.0
28
+
29
+
30
+ def _ping(celery) -> None:
31
+ """Open and close a broker connection. Raises on any failure."""
32
+ connection = celery.connection()
33
+ try:
34
+ connection.ensure_connection(max_retries=0, timeout=_CONNECT_TIMEOUT_SECONDS)
35
+ finally:
36
+ connection.release()
37
+
38
+
39
+ def build_redis_check(app: FastAPI):
40
+ """Return an async check closing over *app* so it re-reads live settings."""
41
+
42
+ async def check() -> HealthCheckResult:
43
+ services = getattr(app.state, "background_tasks", None)
44
+ celery = getattr(services, "celery", None)
45
+ if celery is None:
46
+ return HealthCheckResult(
47
+ status=HealthStatus.UNHEALTHY, detail="Celery app not initialised"
48
+ )
49
+
50
+ try:
51
+ await asyncio.to_thread(_ping, celery)
52
+ except Exception as exc:
53
+ return HealthCheckResult(status=HealthStatus.UNHEALTHY, detail=str(exc))
54
+
55
+ broker = getattr(services.settings, "broker_url", "")
56
+ return HealthCheckResult(status=HealthStatus.HEALTHY, detail=f"Broker reachable ({broker})")
57
+
58
+ return check
@@ -0,0 +1,152 @@
1
+ {
2
+ "index": {
3
+ "title": "Background tasks",
4
+ "description": "Monitor executions and retry failed or stuck jobs",
5
+ "search_placeholder": "Search by task name…",
6
+ "workers_button": "Workers",
7
+ "retry_all_button": "Retry all failed",
8
+ "previous": "Previous",
9
+ "next": "Next",
10
+ "showing": "Showing {from}–{to} of {total}"
11
+ },
12
+ "strip": {
13
+ "queued": "Queued",
14
+ "running": "Running",
15
+ "succeeded": "Succeeded",
16
+ "failed": "Failed",
17
+ "stuck": "Stuck",
18
+ "window_all": "All time",
19
+ "window_24h": "Last 24h"
20
+ },
21
+ "filters": {
22
+ "status_label": "Filter by status",
23
+ "all": "All",
24
+ "failed": "Failed",
25
+ "running": "Running",
26
+ "stuck": "Stuck",
27
+ "queue_label": "Queue:",
28
+ "queue_all": "all",
29
+ "queue_aria": "Filter by queue"
30
+ },
31
+ "status": {
32
+ "queued": "queued",
33
+ "running": "running",
34
+ "success": "success",
35
+ "failed": "failed",
36
+ "stuck": "stuck",
37
+ "revoked": "revoked",
38
+ "retrying": "retrying"
39
+ },
40
+ "table": {
41
+ "task": "Task",
42
+ "status": "Status",
43
+ "queue": "Queue",
44
+ "queued_at": "Queued",
45
+ "duration": "Duration",
46
+ "actions": "Actions",
47
+ "retry": "Retry"
48
+ },
49
+ "detail": {
50
+ "head_title": "Task detail",
51
+ "description": "execution {id} · attempt {attempt} of {max}",
52
+ "attempt": "attempt {attempt} of {max}",
53
+ "back_button": "Back to executions",
54
+ "retry_button": "Retry task",
55
+ "meta": "Details",
56
+ "queue": "Queue",
57
+ "worker": "Worker",
58
+ "celery_id": "Celery id",
59
+ "queued_at": "Queued at",
60
+ "started_at": "Started at",
61
+ "finished_at": "Finished at",
62
+ "duration": "Duration",
63
+ "heartbeat": "Heartbeat",
64
+ "retried_from": "Retried from",
65
+ "exception": "Exception",
66
+ "args": "Arguments",
67
+ "kwargs": "Keyword arguments",
68
+ "result": "Result",
69
+ "traceback": "Traceback",
70
+ "copy": "Copy",
71
+ "copied": "Copied",
72
+ "no_traceback": "No traceback recorded."
73
+ },
74
+ "retry_dialog": {
75
+ "title": "Retry {name}?",
76
+ "description": "A new execution is queued with the same arguments.",
77
+ "retried_before_one": "This one has already been retried once.",
78
+ "retried_before_other": "This one has already been retried {count} times.",
79
+ "args_label": "args",
80
+ "kwargs_label": "kwargs",
81
+ "no_args": "This task takes no arguments.",
82
+ "cancel": "Cancel",
83
+ "confirm": "Queue retry"
84
+ },
85
+ "retry_all_dialog": {
86
+ "title": "Retry all failed tasks?",
87
+ "description": "Every failed or stuck execution in the current view is queued again with its original arguments. Rows that already have a retry in flight are skipped.",
88
+ "scope_all_queues": "All queues",
89
+ "scope_queue": "Queue {queue}",
90
+ "scope_all_statuses": "Failed and stuck",
91
+ "cancel": "Cancel",
92
+ "confirm": "Queue retries",
93
+ "scope_search": "Search “{query}”",
94
+ "scope_no_search": "Any task name"
95
+ },
96
+ "toasts": {
97
+ "retried": "Task \"{name}\" re-enqueued",
98
+ "retry_failed": "Failed to retry task",
99
+ "bulk_retried_one": "Queued {count} retry",
100
+ "bulk_retried_other": "Queued {count} retries",
101
+ "bulk_retry_failed": "Failed to retry tasks",
102
+ "bulk_remaining_one": "{count} more remains — run it again.",
103
+ "bulk_remaining_other": "{count} more remain — run it again."
104
+ },
105
+ "worker_health": {
106
+ "broker_unreachable_title": "Broker unreachable",
107
+ "broker_unreachable_detail": "Nothing can be queued or run until the broker is back. The counts below are the last state written to the database.",
108
+ "no_workers_title_one": "{count} task waiting, no worker running",
109
+ "no_workers_title_other": "{count} tasks waiting, no worker running",
110
+ "no_workers_detail": "The broker is up but nothing is consuming the queue, so this backlog will not move on its own.",
111
+ "view_workers": "View workers"
112
+ },
113
+ "tasks_empty": {
114
+ "filtered_title": "No tasks match these filters",
115
+ "filtered_description": "Executions exist outside the current search, status and queue filters.",
116
+ "clear_filters": "Clear filters",
117
+ "broker_unreachable_title": "Can't reach the broker",
118
+ "broker_unreachable_description": "Nothing can be queued or executed until the broker is back. Tasks are not being lost — they are not being accepted.",
119
+ "no_workers_title": "No worker is running",
120
+ "no_workers_description": "The broker is up, but nothing is consuming the queue. Tasks enqueued now will wait rather than run.",
121
+ "healthy_title": "No task has run yet",
122
+ "healthy_description": "Registered tasks appear here the first time they execute."
123
+ },
124
+ "workers": {
125
+ "title": "Workers",
126
+ "description": "Celery workers connected to the broker",
127
+ "back_button": "Executions",
128
+ "last_updated": "Last updated {time}",
129
+ "refresh": "Refresh",
130
+ "online": "Online",
131
+ "offline": "Offline",
132
+ "uptime": "uptime {uptime}",
133
+ "offline_state": "offline",
134
+ "active": "Active",
135
+ "pool": "Pool",
136
+ "processed": "Processed",
137
+ "queues": "Queues",
138
+ "broker_unreachable_title": "Broker unreachable",
139
+ "broker_unreachable_description": "The connection to the broker failed, so nothing can be queued or run. Check this setting and confirm the broker is up.",
140
+ "no_error_reported": "No error message reported.",
141
+ "no_workers_title": "No workers connected",
142
+ "no_workers_description": "The broker is reachable but nothing is consuming the queue. Start a worker locally with:",
143
+ "refresh_failed_status": "Failed to refresh workers (HTTP {status})",
144
+ "refresh_failed": "Failed to refresh workers"
145
+ },
146
+ "nav": {
147
+ "background_tasks": "Background tasks"
148
+ },
149
+ "audit": {
150
+ "task_execution": "Task execution"
151
+ }
152
+ }