qodalis-cli-server-plugin-jobs 1.0.0b3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qodalis_cli_server_plugin_jobs-1.0.0b3/.gitignore +43 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/PKG-INFO +22 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/pyproject.toml +41 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/__init__.py +22 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/cli_job_execution_context.py +21 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/cli_job_logger.py +52 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/cli_job_scheduler.py +472 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/cli_jobs_builder.py +54 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/cli_jobs_controller.py +265 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/in_memory_job_storage_provider.py +56 -0
- qodalis_cli_server_plugin_jobs-1.0.0b3/qodalis_cli_jobs/interval_parser.py +29 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Byte-compiled / optimized
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
*.egg-info/
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
.eggs/
|
|
11
|
+
*.egg
|
|
12
|
+
|
|
13
|
+
# Virtual environments
|
|
14
|
+
.venv/
|
|
15
|
+
venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Environment variables
|
|
19
|
+
.env
|
|
20
|
+
|
|
21
|
+
# Testing
|
|
22
|
+
.pytest_cache/
|
|
23
|
+
.coverage
|
|
24
|
+
htmlcov/
|
|
25
|
+
TestResults/
|
|
26
|
+
|
|
27
|
+
# Type checking / linting
|
|
28
|
+
.mypy_cache/
|
|
29
|
+
.ruff_cache/
|
|
30
|
+
|
|
31
|
+
# C extensions
|
|
32
|
+
*.so
|
|
33
|
+
|
|
34
|
+
# IDEs and editors
|
|
35
|
+
.idea/
|
|
36
|
+
.vscode/*
|
|
37
|
+
!.vscode/settings.json
|
|
38
|
+
!.vscode/launch.json
|
|
39
|
+
!.vscode/extensions.json
|
|
40
|
+
|
|
41
|
+
# System files
|
|
42
|
+
.DS_Store
|
|
43
|
+
Thumbs.db
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: qodalis-cli-server-plugin-jobs
|
|
3
|
+
Version: 1.0.0b3
|
|
4
|
+
Summary: Background job scheduler plugin for Qodalis CLI servers
|
|
5
|
+
Project-URL: Homepage, https://github.com/qodalis-solutions/cli-server-python
|
|
6
|
+
Project-URL: Repository, https://github.com/qodalis-solutions/cli-server-python
|
|
7
|
+
Project-URL: Issues, https://github.com/qodalis-solutions/cli-server-python/issues
|
|
8
|
+
Author-email: Nicolae Lupei <nicolae.lupei@qodalis.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Keywords: cli,jobs,plugin,qodalis,scheduler
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: croniter>=1.3.0
|
|
21
|
+
Requires-Dist: fastapi>=0.100.0
|
|
22
|
+
Requires-Dist: qodalis-cli-server-abstractions
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "qodalis-cli-server-plugin-jobs"
|
|
7
|
+
version = "1.0.0b3"
|
|
8
|
+
description = "Background job scheduler plugin for Qodalis CLI servers"
|
|
9
|
+
readme = { text = "", content-type = "text/plain" }
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Nicolae Lupei", email = "nicolae.lupei@qodalis.com" },
|
|
14
|
+
]
|
|
15
|
+
keywords = ["cli", "qodalis", "jobs", "scheduler", "plugin"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"qodalis-cli-server-abstractions",
|
|
28
|
+
"croniter>=1.3.0",
|
|
29
|
+
"fastapi>=0.100.0",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://github.com/qodalis-solutions/cli-server-python"
|
|
34
|
+
Repository = "https://github.com/qodalis-solutions/cli-server-python"
|
|
35
|
+
Issues = "https://github.com/qodalis-solutions/cli-server-python/issues"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["qodalis_cli_jobs"]
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.sdist]
|
|
41
|
+
include = ["qodalis_cli_jobs/**"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Qodalis CLI jobs plugin — background job scheduler."""
|
|
2
|
+
|
|
3
|
+
from .cli_job_execution_context import CliJobExecutionContext
|
|
4
|
+
from .cli_job_logger import CliJobLogger
|
|
5
|
+
from .cli_job_scheduler import CliJobScheduler, InvalidOperationError, BroadcastFn
|
|
6
|
+
from .cli_jobs_builder import CliJobsBuilder, CliJobsPlugin
|
|
7
|
+
from .cli_jobs_controller import create_cli_jobs_router
|
|
8
|
+
from .in_memory_job_storage_provider import InMemoryJobStorageProvider
|
|
9
|
+
from .interval_parser import parse_interval
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"BroadcastFn",
|
|
13
|
+
"CliJobExecutionContext",
|
|
14
|
+
"CliJobLogger",
|
|
15
|
+
"CliJobScheduler",
|
|
16
|
+
"CliJobsBuilder",
|
|
17
|
+
"CliJobsPlugin",
|
|
18
|
+
"InMemoryJobStorageProvider",
|
|
19
|
+
"InvalidOperationError",
|
|
20
|
+
"create_cli_jobs_router",
|
|
21
|
+
"parse_interval",
|
|
22
|
+
]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from qodalis_cli_server_abstractions.jobs import ICliJobExecutionContext, ICliJobLogger
|
|
4
|
+
|
|
5
|
+
from .cli_job_logger import CliJobLogger
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CliJobExecutionContext(ICliJobExecutionContext):
|
|
9
|
+
"""Concrete execution context wrapping a per-execution logger."""
|
|
10
|
+
|
|
11
|
+
def __init__(self) -> None:
|
|
12
|
+
self._logger = CliJobLogger()
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def logger(self) -> ICliJobLogger:
|
|
16
|
+
return self._logger
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def log_entries(self) -> list:
|
|
20
|
+
"""Convenience accessor for the captured log entries."""
|
|
21
|
+
return self._logger.entries
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
|
|
5
|
+
from qodalis_cli_server_abstractions.jobs import ICliJobLogger, JobLogEntry
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CliJobLogger(ICliJobLogger):
|
|
9
|
+
"""Concrete logger that captures log entries in a list."""
|
|
10
|
+
|
|
11
|
+
def __init__(self) -> None:
|
|
12
|
+
self._entries: list[JobLogEntry] = []
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def entries(self) -> list[JobLogEntry]:
|
|
16
|
+
return self._entries
|
|
17
|
+
|
|
18
|
+
def debug(self, message: str) -> None:
|
|
19
|
+
self._entries.append(
|
|
20
|
+
JobLogEntry(
|
|
21
|
+
timestamp=datetime.datetime.now(datetime.UTC),
|
|
22
|
+
level="debug",
|
|
23
|
+
message=message,
|
|
24
|
+
)
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
def info(self, message: str) -> None:
|
|
28
|
+
self._entries.append(
|
|
29
|
+
JobLogEntry(
|
|
30
|
+
timestamp=datetime.datetime.now(datetime.UTC),
|
|
31
|
+
level="info",
|
|
32
|
+
message=message,
|
|
33
|
+
)
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
def warning(self, message: str) -> None:
|
|
37
|
+
self._entries.append(
|
|
38
|
+
JobLogEntry(
|
|
39
|
+
timestamp=datetime.datetime.now(datetime.UTC),
|
|
40
|
+
level="warning",
|
|
41
|
+
message=message,
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def error(self, message: str) -> None:
|
|
46
|
+
self._entries.append(
|
|
47
|
+
JobLogEntry(
|
|
48
|
+
timestamp=datetime.datetime.now(datetime.UTC),
|
|
49
|
+
level="error",
|
|
50
|
+
message=message,
|
|
51
|
+
)
|
|
52
|
+
)
|
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import datetime
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import uuid
|
|
8
|
+
from collections import deque
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any, Callable, Awaitable
|
|
11
|
+
|
|
12
|
+
from croniter import croniter
|
|
13
|
+
|
|
14
|
+
from qodalis_cli_server_abstractions.jobs import (
|
|
15
|
+
CliJobOptions,
|
|
16
|
+
ICliJob,
|
|
17
|
+
ICliJobStorageProvider,
|
|
18
|
+
JobExecution,
|
|
19
|
+
JobState,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
from .cli_job_execution_context import CliJobExecutionContext
|
|
23
|
+
from .interval_parser import parse_interval
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class _JobRegistration:
|
|
30
|
+
"""Internal runtime state for a registered job."""
|
|
31
|
+
|
|
32
|
+
id: str
|
|
33
|
+
job: ICliJob
|
|
34
|
+
options: CliJobOptions
|
|
35
|
+
status: str = "active" # 'active' | 'paused' | 'stopped'
|
|
36
|
+
current_execution_id: str | None = None
|
|
37
|
+
current_cancellation: asyncio.Event | None = None
|
|
38
|
+
next_run_at: datetime.datetime | None = None
|
|
39
|
+
last_run_at: datetime.datetime | None = None
|
|
40
|
+
last_run_status: str | None = None
|
|
41
|
+
last_run_duration: float | None = None # ms
|
|
42
|
+
timer_task: asyncio.Task[Any] | None = None
|
|
43
|
+
queue: deque[int] = field(default_factory=deque) # queued retry_attempt values
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# Type alias for the broadcast callback.
|
|
47
|
+
BroadcastFn = Callable[[str], Awaitable[None]]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class CliJobScheduler:
|
|
51
|
+
"""Background job scheduler using asyncio tasks."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
storage: ICliJobStorageProvider,
|
|
56
|
+
broadcast_fn: BroadcastFn | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
self._storage = storage
|
|
59
|
+
self._broadcast_fn = broadcast_fn
|
|
60
|
+
self._registrations: dict[str, _JobRegistration] = {}
|
|
61
|
+
self._running = False
|
|
62
|
+
|
|
63
|
+
# ------------------------------------------------------------------
|
|
64
|
+
# Public properties
|
|
65
|
+
# ------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def registrations(self) -> dict[str, _JobRegistration]:
|
|
69
|
+
return self._registrations
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def storage(self) -> ICliJobStorageProvider:
|
|
73
|
+
return self._storage
|
|
74
|
+
|
|
75
|
+
# ------------------------------------------------------------------
|
|
76
|
+
# Registration
|
|
77
|
+
# ------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
def register(self, job: ICliJob, options: CliJobOptions) -> str:
|
|
80
|
+
"""Register a job. Returns the job id."""
|
|
81
|
+
job_id = str(uuid.uuid4())
|
|
82
|
+
if not options.name:
|
|
83
|
+
options.name = type(job).__name__
|
|
84
|
+
if not options.description:
|
|
85
|
+
options.description = options.name
|
|
86
|
+
reg = _JobRegistration(
|
|
87
|
+
id=job_id,
|
|
88
|
+
job=job,
|
|
89
|
+
options=options,
|
|
90
|
+
status="active" if options.enabled else "stopped",
|
|
91
|
+
)
|
|
92
|
+
self._registrations[job_id] = reg
|
|
93
|
+
return job_id
|
|
94
|
+
|
|
95
|
+
# ------------------------------------------------------------------
|
|
96
|
+
# Lifecycle
|
|
97
|
+
# ------------------------------------------------------------------
|
|
98
|
+
|
|
99
|
+
async def start(self) -> None:
|
|
100
|
+
"""Load persisted states and start timers for active jobs."""
|
|
101
|
+
self._running = True
|
|
102
|
+
persisted = await self._storage.get_all_job_states()
|
|
103
|
+
|
|
104
|
+
for reg in self._registrations.values():
|
|
105
|
+
# Apply persisted state if available
|
|
106
|
+
state = persisted.get(reg.id)
|
|
107
|
+
if state:
|
|
108
|
+
reg.status = state.status
|
|
109
|
+
reg.last_run_at = state.last_run_at
|
|
110
|
+
|
|
111
|
+
if reg.status == "active":
|
|
112
|
+
self._schedule_next(reg)
|
|
113
|
+
|
|
114
|
+
async def stop(self) -> None:
|
|
115
|
+
"""Cancel all running executions, stop timers, persist states."""
|
|
116
|
+
self._running = False
|
|
117
|
+
for reg in list(self._registrations.values()):
|
|
118
|
+
# Cancel running execution
|
|
119
|
+
if reg.current_cancellation is not None:
|
|
120
|
+
reg.current_cancellation.set()
|
|
121
|
+
# Cancel timer
|
|
122
|
+
if reg.timer_task is not None and not reg.timer_task.done():
|
|
123
|
+
reg.timer_task.cancel()
|
|
124
|
+
try:
|
|
125
|
+
await reg.timer_task
|
|
126
|
+
except (asyncio.CancelledError, Exception):
|
|
127
|
+
pass
|
|
128
|
+
# Persist state
|
|
129
|
+
await self._storage.save_job_state(
|
|
130
|
+
reg.id,
|
|
131
|
+
JobState(
|
|
132
|
+
status=reg.status,
|
|
133
|
+
last_run_at=reg.last_run_at,
|
|
134
|
+
updated_at=datetime.datetime.now(datetime.UTC),
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# ------------------------------------------------------------------
|
|
139
|
+
# Control methods
|
|
140
|
+
# ------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
async def trigger(self, job_id: str) -> None:
|
|
143
|
+
reg = self._get_registration(job_id)
|
|
144
|
+
if reg.current_execution_id is not None:
|
|
145
|
+
policy = reg.options.overlap_policy
|
|
146
|
+
if policy == "skip":
|
|
147
|
+
raise InvalidOperationError("Job is already running")
|
|
148
|
+
elif policy == "cancel":
|
|
149
|
+
if reg.current_cancellation:
|
|
150
|
+
reg.current_cancellation.set()
|
|
151
|
+
elif policy == "queue":
|
|
152
|
+
reg.queue.append(0)
|
|
153
|
+
return
|
|
154
|
+
asyncio.create_task(self._execute_job(reg, retry_attempt=0))
|
|
155
|
+
|
|
156
|
+
async def pause(self, job_id: str) -> None:
|
|
157
|
+
reg = self._get_registration(job_id)
|
|
158
|
+
if reg.status == "paused":
|
|
159
|
+
raise InvalidOperationError("Job is already paused")
|
|
160
|
+
reg.status = "paused"
|
|
161
|
+
self._cancel_timer(reg)
|
|
162
|
+
await self._persist_state(reg)
|
|
163
|
+
await self._broadcast({"type": "job:paused", "jobId": job_id})
|
|
164
|
+
|
|
165
|
+
async def resume(self, job_id: str) -> None:
|
|
166
|
+
reg = self._get_registration(job_id)
|
|
167
|
+
if reg.status != "paused":
|
|
168
|
+
raise InvalidOperationError("Job is not paused")
|
|
169
|
+
reg.status = "active"
|
|
170
|
+
self._schedule_next(reg)
|
|
171
|
+
await self._persist_state(reg)
|
|
172
|
+
await self._broadcast({"type": "job:resumed", "jobId": job_id})
|
|
173
|
+
|
|
174
|
+
async def stop_job(self, job_id: str) -> None:
|
|
175
|
+
reg = self._get_registration(job_id)
|
|
176
|
+
reg.status = "stopped"
|
|
177
|
+
self._cancel_timer(reg)
|
|
178
|
+
if reg.current_cancellation is not None:
|
|
179
|
+
reg.current_cancellation.set()
|
|
180
|
+
await self._persist_state(reg)
|
|
181
|
+
await self._broadcast({"type": "job:stopped", "jobId": job_id})
|
|
182
|
+
|
|
183
|
+
async def cancel_current(self, job_id: str) -> None:
|
|
184
|
+
reg = self._get_registration(job_id)
|
|
185
|
+
if reg.current_execution_id is None or reg.current_cancellation is None:
|
|
186
|
+
raise InvalidOperationError("No execution is currently running")
|
|
187
|
+
reg.current_cancellation.set()
|
|
188
|
+
|
|
189
|
+
async def update_options(
|
|
190
|
+
self,
|
|
191
|
+
job_id: str,
|
|
192
|
+
*,
|
|
193
|
+
description: str | None = None,
|
|
194
|
+
group: str | None = None,
|
|
195
|
+
schedule: str | None = None,
|
|
196
|
+
interval: str | None = None,
|
|
197
|
+
max_retries: int | None = None,
|
|
198
|
+
retry_delay: str | None = None,
|
|
199
|
+
retry_strategy: str | None = None,
|
|
200
|
+
timeout: str | None = None,
|
|
201
|
+
overlap_policy: str | None = None,
|
|
202
|
+
) -> None:
|
|
203
|
+
reg = self._get_registration(job_id)
|
|
204
|
+
opts = reg.options
|
|
205
|
+
|
|
206
|
+
if description is not None:
|
|
207
|
+
opts.description = description
|
|
208
|
+
if group is not None:
|
|
209
|
+
opts.group = group
|
|
210
|
+
if schedule is not None and interval is not None:
|
|
211
|
+
raise ValueError("Cannot provide both schedule and interval")
|
|
212
|
+
if schedule is not None:
|
|
213
|
+
if not croniter.is_valid(schedule):
|
|
214
|
+
raise ValueError(f"Invalid cron expression: {schedule!r}")
|
|
215
|
+
opts.schedule = schedule
|
|
216
|
+
opts.interval = None
|
|
217
|
+
if interval is not None:
|
|
218
|
+
parse_interval(interval) # validate
|
|
219
|
+
opts.interval = interval
|
|
220
|
+
opts.schedule = None
|
|
221
|
+
if max_retries is not None:
|
|
222
|
+
opts.max_retries = max_retries
|
|
223
|
+
if retry_delay is not None:
|
|
224
|
+
parse_interval(retry_delay) # validate
|
|
225
|
+
opts.retry_delay = retry_delay
|
|
226
|
+
if retry_strategy is not None:
|
|
227
|
+
if retry_strategy not in ("fixed", "linear", "exponential"):
|
|
228
|
+
raise ValueError(f"Invalid retry strategy: {retry_strategy!r}")
|
|
229
|
+
opts.retry_strategy = retry_strategy
|
|
230
|
+
if timeout is not None:
|
|
231
|
+
parse_interval(timeout) # validate
|
|
232
|
+
opts.timeout = timeout
|
|
233
|
+
if overlap_policy is not None:
|
|
234
|
+
if overlap_policy not in ("skip", "queue", "cancel"):
|
|
235
|
+
raise ValueError(f"Invalid overlap policy: {overlap_policy!r}")
|
|
236
|
+
opts.overlap_policy = overlap_policy
|
|
237
|
+
|
|
238
|
+
# Reschedule if active
|
|
239
|
+
if reg.status == "active":
|
|
240
|
+
self._cancel_timer(reg)
|
|
241
|
+
self._schedule_next(reg)
|
|
242
|
+
|
|
243
|
+
# ------------------------------------------------------------------
|
|
244
|
+
# Internal scheduling
|
|
245
|
+
# ------------------------------------------------------------------
|
|
246
|
+
|
|
247
|
+
def _schedule_next(self, reg: _JobRegistration) -> None:
|
|
248
|
+
"""Compute next run time and create timer task."""
|
|
249
|
+
if not self._running:
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
now = datetime.datetime.now(datetime.UTC)
|
|
253
|
+
delay: float
|
|
254
|
+
|
|
255
|
+
if reg.options.schedule:
|
|
256
|
+
cron = croniter(reg.options.schedule, now)
|
|
257
|
+
next_dt = cron.get_next(datetime.datetime)
|
|
258
|
+
if next_dt.tzinfo is None:
|
|
259
|
+
next_dt = next_dt.replace(tzinfo=datetime.UTC)
|
|
260
|
+
reg.next_run_at = next_dt
|
|
261
|
+
delay = max((next_dt - now).total_seconds(), 0.1)
|
|
262
|
+
elif reg.options.interval:
|
|
263
|
+
delay = parse_interval(reg.options.interval)
|
|
264
|
+
reg.next_run_at = now + datetime.timedelta(seconds=delay)
|
|
265
|
+
else:
|
|
266
|
+
# No schedule configured — job can only be triggered manually
|
|
267
|
+
reg.next_run_at = None
|
|
268
|
+
return
|
|
269
|
+
|
|
270
|
+
reg.timer_task = asyncio.create_task(self._timer_fire(reg, delay))
|
|
271
|
+
|
|
272
|
+
async def _timer_fire(self, reg: _JobRegistration, delay: float) -> None:
|
|
273
|
+
"""Wait for the delay, then execute the job and reschedule."""
|
|
274
|
+
try:
|
|
275
|
+
await asyncio.sleep(delay)
|
|
276
|
+
except asyncio.CancelledError:
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
if not self._running or reg.status != "active":
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
# Overlap check
|
|
283
|
+
if reg.current_execution_id is not None:
|
|
284
|
+
policy = reg.options.overlap_policy
|
|
285
|
+
if policy == "skip":
|
|
286
|
+
pass # skip this firing
|
|
287
|
+
elif policy == "cancel":
|
|
288
|
+
if reg.current_cancellation:
|
|
289
|
+
reg.current_cancellation.set()
|
|
290
|
+
asyncio.create_task(self._execute_job(reg, retry_attempt=0))
|
|
291
|
+
elif policy == "queue":
|
|
292
|
+
reg.queue.append(0)
|
|
293
|
+
else:
|
|
294
|
+
asyncio.create_task(self._execute_job(reg, retry_attempt=0))
|
|
295
|
+
|
|
296
|
+
# Reschedule
|
|
297
|
+
if reg.status == "active":
|
|
298
|
+
self._schedule_next(reg)
|
|
299
|
+
|
|
300
|
+
# ------------------------------------------------------------------
|
|
301
|
+
# Job execution
|
|
302
|
+
# ------------------------------------------------------------------
|
|
303
|
+
|
|
304
|
+
async def _execute_job(self, reg: _JobRegistration, retry_attempt: int) -> None:
|
|
305
|
+
exec_id = str(uuid.uuid4())
|
|
306
|
+
cancel_event = asyncio.Event()
|
|
307
|
+
now = datetime.datetime.now(datetime.UTC)
|
|
308
|
+
|
|
309
|
+
reg.current_execution_id = exec_id
|
|
310
|
+
reg.current_cancellation = cancel_event
|
|
311
|
+
|
|
312
|
+
execution = JobExecution(
|
|
313
|
+
id=exec_id,
|
|
314
|
+
job_id=reg.id,
|
|
315
|
+
job_name=reg.options.name or type(reg.job).__name__,
|
|
316
|
+
status="running",
|
|
317
|
+
started_at=now,
|
|
318
|
+
retry_attempt=retry_attempt,
|
|
319
|
+
)
|
|
320
|
+
await self._storage.save_execution(execution)
|
|
321
|
+
|
|
322
|
+
await self._broadcast({
|
|
323
|
+
"type": "job:started",
|
|
324
|
+
"jobId": reg.id,
|
|
325
|
+
"executionId": exec_id,
|
|
326
|
+
"timestamp": now.isoformat(),
|
|
327
|
+
})
|
|
328
|
+
|
|
329
|
+
ctx = CliJobExecutionContext()
|
|
330
|
+
|
|
331
|
+
try:
|
|
332
|
+
timeout_seconds: float | None = None
|
|
333
|
+
if reg.options.timeout:
|
|
334
|
+
timeout_seconds = parse_interval(reg.options.timeout)
|
|
335
|
+
|
|
336
|
+
if timeout_seconds:
|
|
337
|
+
await asyncio.wait_for(
|
|
338
|
+
reg.job.execute_async(ctx, cancel_event),
|
|
339
|
+
timeout=timeout_seconds,
|
|
340
|
+
)
|
|
341
|
+
else:
|
|
342
|
+
await reg.job.execute_async(ctx, cancel_event)
|
|
343
|
+
|
|
344
|
+
# Check if cancellation was requested (cooperative)
|
|
345
|
+
if cancel_event.is_set():
|
|
346
|
+
execution.status = "cancelled"
|
|
347
|
+
await self._broadcast({
|
|
348
|
+
"type": "job:cancelled",
|
|
349
|
+
"jobId": reg.id,
|
|
350
|
+
"executionId": exec_id,
|
|
351
|
+
})
|
|
352
|
+
else:
|
|
353
|
+
execution.status = "completed"
|
|
354
|
+
|
|
355
|
+
except asyncio.TimeoutError:
|
|
356
|
+
cancel_event.set()
|
|
357
|
+
execution.status = "timed_out"
|
|
358
|
+
execution.error = f"Timed out after {reg.options.timeout}"
|
|
359
|
+
await self._broadcast({
|
|
360
|
+
"type": "job:timed_out",
|
|
361
|
+
"jobId": reg.id,
|
|
362
|
+
"executionId": exec_id,
|
|
363
|
+
"timeout": reg.options.timeout,
|
|
364
|
+
})
|
|
365
|
+
except asyncio.CancelledError:
|
|
366
|
+
execution.status = "cancelled"
|
|
367
|
+
await self._broadcast({
|
|
368
|
+
"type": "job:cancelled",
|
|
369
|
+
"jobId": reg.id,
|
|
370
|
+
"executionId": exec_id,
|
|
371
|
+
})
|
|
372
|
+
except Exception as exc:
|
|
373
|
+
execution.status = "failed"
|
|
374
|
+
execution.error = str(exc)
|
|
375
|
+
await self._broadcast({
|
|
376
|
+
"type": "job:failed",
|
|
377
|
+
"jobId": reg.id,
|
|
378
|
+
"executionId": exec_id,
|
|
379
|
+
"error": str(exc),
|
|
380
|
+
})
|
|
381
|
+
|
|
382
|
+
# Finalize
|
|
383
|
+
end_time = datetime.datetime.now(datetime.UTC)
|
|
384
|
+
execution.completed_at = end_time
|
|
385
|
+
execution.duration = round((end_time - execution.started_at).total_seconds() * 1000)
|
|
386
|
+
execution.logs = list(ctx.log_entries)
|
|
387
|
+
await self._storage.save_execution(execution)
|
|
388
|
+
|
|
389
|
+
reg.last_run_at = execution.started_at
|
|
390
|
+
reg.last_run_status = execution.status
|
|
391
|
+
reg.last_run_duration = execution.duration
|
|
392
|
+
reg.current_execution_id = None
|
|
393
|
+
reg.current_cancellation = None
|
|
394
|
+
|
|
395
|
+
if execution.status == "completed":
|
|
396
|
+
await self._broadcast({
|
|
397
|
+
"type": "job:completed",
|
|
398
|
+
"jobId": reg.id,
|
|
399
|
+
"executionId": exec_id,
|
|
400
|
+
"duration": execution.duration,
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
# Retry on failure
|
|
404
|
+
if execution.status == "failed" and retry_attempt < reg.options.max_retries:
|
|
405
|
+
base_delay = parse_interval(reg.options.retry_delay) if reg.options.retry_delay else 5.0
|
|
406
|
+
strategy = reg.options.retry_strategy or "exponential"
|
|
407
|
+
if strategy == "fixed":
|
|
408
|
+
delay = base_delay
|
|
409
|
+
elif strategy == "linear":
|
|
410
|
+
delay = base_delay * (retry_attempt + 1)
|
|
411
|
+
else: # exponential
|
|
412
|
+
delay = base_delay * (2 ** retry_attempt)
|
|
413
|
+
logger.info(
|
|
414
|
+
"Retrying job %s in %.1fs (attempt %d/%d)",
|
|
415
|
+
reg.options.name,
|
|
416
|
+
delay,
|
|
417
|
+
retry_attempt + 1,
|
|
418
|
+
reg.options.max_retries,
|
|
419
|
+
)
|
|
420
|
+
await self._broadcast({
|
|
421
|
+
"type": "job:retrying",
|
|
422
|
+
"jobId": reg.id,
|
|
423
|
+
"executionId": exec_id,
|
|
424
|
+
"attempt": retry_attempt + 1,
|
|
425
|
+
"maxRetries": reg.options.max_retries,
|
|
426
|
+
"delayMs": round(delay * 1000),
|
|
427
|
+
})
|
|
428
|
+
await asyncio.sleep(delay)
|
|
429
|
+
asyncio.create_task(self._execute_job(reg, retry_attempt + 1))
|
|
430
|
+
return
|
|
431
|
+
|
|
432
|
+
# Process queue
|
|
433
|
+
if reg.queue:
|
|
434
|
+
queued_attempt = reg.queue.popleft()
|
|
435
|
+
asyncio.create_task(self._execute_job(reg, queued_attempt))
|
|
436
|
+
|
|
437
|
+
# ------------------------------------------------------------------
|
|
438
|
+
# Helpers
|
|
439
|
+
# ------------------------------------------------------------------
|
|
440
|
+
|
|
441
|
+
def _get_registration(self, job_id: str) -> _JobRegistration:
|
|
442
|
+
reg = self._registrations.get(job_id)
|
|
443
|
+
if reg is None:
|
|
444
|
+
raise KeyError(f"Job not found: {job_id}")
|
|
445
|
+
return reg
|
|
446
|
+
|
|
447
|
+
def _cancel_timer(self, reg: _JobRegistration) -> None:
|
|
448
|
+
if reg.timer_task is not None and not reg.timer_task.done():
|
|
449
|
+
reg.timer_task.cancel()
|
|
450
|
+
reg.timer_task = None
|
|
451
|
+
reg.next_run_at = None
|
|
452
|
+
|
|
453
|
+
async def _persist_state(self, reg: _JobRegistration) -> None:
|
|
454
|
+
await self._storage.save_job_state(
|
|
455
|
+
reg.id,
|
|
456
|
+
JobState(
|
|
457
|
+
status=reg.status,
|
|
458
|
+
last_run_at=reg.last_run_at,
|
|
459
|
+
updated_at=datetime.datetime.now(datetime.UTC),
|
|
460
|
+
),
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
async def _broadcast(self, message: dict[str, Any]) -> None:
|
|
464
|
+
if self._broadcast_fn is not None:
|
|
465
|
+
try:
|
|
466
|
+
await self._broadcast_fn(json.dumps(message))
|
|
467
|
+
except Exception:
|
|
468
|
+
logger.debug("Failed to broadcast job event", exc_info=True)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
class InvalidOperationError(Exception):
|
|
472
|
+
"""Raised when a job control operation is not valid in the current state."""
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Callable, Awaitable
|
|
5
|
+
|
|
6
|
+
from fastapi import APIRouter
|
|
7
|
+
|
|
8
|
+
from qodalis_cli_server_abstractions.jobs import (
|
|
9
|
+
CliJobOptions,
|
|
10
|
+
ICliJob,
|
|
11
|
+
ICliJobStorageProvider,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from .cli_job_scheduler import CliJobScheduler, BroadcastFn
|
|
15
|
+
from .cli_jobs_controller import create_cli_jobs_router
|
|
16
|
+
from .in_memory_job_storage_provider import InMemoryJobStorageProvider
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class CliJobsPlugin:
|
|
21
|
+
"""Result of building the jobs plugin — contains the router and scheduler."""
|
|
22
|
+
|
|
23
|
+
router: APIRouter
|
|
24
|
+
scheduler: CliJobScheduler
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CliJobsBuilder:
|
|
28
|
+
"""Fluent builder for configuring and creating the jobs plugin."""
|
|
29
|
+
|
|
30
|
+
def __init__(self) -> None:
|
|
31
|
+
self._registrations: list[tuple[ICliJob, CliJobOptions]] = []
|
|
32
|
+
self._storage_provider: ICliJobStorageProvider | None = None
|
|
33
|
+
|
|
34
|
+
def add_job(self, job: ICliJob, options: CliJobOptions) -> CliJobsBuilder:
|
|
35
|
+
"""Register a background job with the given options."""
|
|
36
|
+
self._registrations.append((job, options))
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def set_storage_provider(self, provider: ICliJobStorageProvider) -> CliJobsBuilder:
|
|
40
|
+
"""Set a custom storage provider for job execution history."""
|
|
41
|
+
self._storage_provider = provider
|
|
42
|
+
return self
|
|
43
|
+
|
|
44
|
+
def build(self, broadcast_fn: BroadcastFn | None = None) -> CliJobsPlugin:
|
|
45
|
+
"""Build the plugin, returning the router and scheduler."""
|
|
46
|
+
storage = self._storage_provider or InMemoryJobStorageProvider()
|
|
47
|
+
scheduler = CliJobScheduler(storage, broadcast_fn=broadcast_fn)
|
|
48
|
+
|
|
49
|
+
for job, opts in self._registrations:
|
|
50
|
+
scheduler.register(job, opts)
|
|
51
|
+
|
|
52
|
+
router = create_cli_jobs_router(scheduler, storage)
|
|
53
|
+
|
|
54
|
+
return CliJobsPlugin(router=router, scheduler=scheduler)
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from fastapi import APIRouter, Query
|
|
7
|
+
from fastapi.responses import JSONResponse
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
from qodalis_cli_server_abstractions.jobs import ICliJobStorageProvider
|
|
11
|
+
|
|
12
|
+
from .cli_job_scheduler import CliJobScheduler, InvalidOperationError
|
|
13
|
+
from .interval_parser import parse_interval
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# Request / Response models
|
|
18
|
+
# ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class UpdateJobRequest(BaseModel):
|
|
22
|
+
description: str | None = None
|
|
23
|
+
group: str | None = None
|
|
24
|
+
schedule: str | None = None
|
|
25
|
+
interval: str | None = None
|
|
26
|
+
max_retries: int | None = Field(None, alias="maxRetries")
|
|
27
|
+
retry_delay: str | None = Field(None, alias="retryDelay")
|
|
28
|
+
retry_strategy: str | None = Field(None, alias="retryStrategy")
|
|
29
|
+
timeout: str | None = None
|
|
30
|
+
overlap_policy: str | None = Field(None, alias="overlapPolicy")
|
|
31
|
+
|
|
32
|
+
model_config = {"populate_by_name": True}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _serialize_dt(dt: datetime.datetime | None) -> str | None:
|
|
36
|
+
if dt is None:
|
|
37
|
+
return None
|
|
38
|
+
return dt.isoformat().replace("+00:00", "Z") if dt.tzinfo else dt.isoformat() + "Z"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _job_dto(reg: Any) -> dict[str, Any]:
|
|
42
|
+
return {
|
|
43
|
+
"id": reg.id,
|
|
44
|
+
"name": reg.options.name,
|
|
45
|
+
"description": reg.options.description,
|
|
46
|
+
"group": reg.options.group,
|
|
47
|
+
"status": reg.status,
|
|
48
|
+
"schedule": reg.options.schedule,
|
|
49
|
+
"interval": reg.options.interval,
|
|
50
|
+
"enabled": reg.status != "stopped",
|
|
51
|
+
"maxRetries": reg.options.max_retries,
|
|
52
|
+
"retryDelay": reg.options.retry_delay,
|
|
53
|
+
"retryStrategy": reg.options.retry_strategy,
|
|
54
|
+
"timeout": reg.options.timeout,
|
|
55
|
+
"overlapPolicy": reg.options.overlap_policy,
|
|
56
|
+
"currentExecutionId": reg.current_execution_id,
|
|
57
|
+
"nextRunAt": _serialize_dt(reg.next_run_at),
|
|
58
|
+
"lastRunAt": _serialize_dt(reg.last_run_at),
|
|
59
|
+
"lastRunStatus": reg.last_run_status,
|
|
60
|
+
"lastRunDuration": reg.last_run_duration,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
# Router factory
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def create_cli_jobs_router(
|
|
70
|
+
scheduler: CliJobScheduler,
|
|
71
|
+
storage: ICliJobStorageProvider,
|
|
72
|
+
) -> APIRouter:
|
|
73
|
+
router = APIRouter()
|
|
74
|
+
|
|
75
|
+
# GET / — list all jobs
|
|
76
|
+
@router.get("")
|
|
77
|
+
async def list_jobs() -> list[dict[str, Any]]:
|
|
78
|
+
return [_job_dto(reg) for reg in scheduler.registrations.values()]
|
|
79
|
+
|
|
80
|
+
# GET /{job_id}
|
|
81
|
+
@router.get("/{job_id}")
|
|
82
|
+
async def get_job(job_id: str) -> Any:
|
|
83
|
+
reg = scheduler.registrations.get(job_id)
|
|
84
|
+
if reg is None:
|
|
85
|
+
return JSONResponse(
|
|
86
|
+
status_code=404,
|
|
87
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
88
|
+
)
|
|
89
|
+
return _job_dto(reg)
|
|
90
|
+
|
|
91
|
+
# POST /{job_id}/trigger
|
|
92
|
+
@router.post("/{job_id}/trigger")
|
|
93
|
+
async def trigger_job(job_id: str) -> Any:
|
|
94
|
+
try:
|
|
95
|
+
await scheduler.trigger(job_id)
|
|
96
|
+
return {"message": "Job triggered"}
|
|
97
|
+
except KeyError:
|
|
98
|
+
return JSONResponse(
|
|
99
|
+
status_code=404,
|
|
100
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
101
|
+
)
|
|
102
|
+
except InvalidOperationError as exc:
|
|
103
|
+
return JSONResponse(
|
|
104
|
+
status_code=409,
|
|
105
|
+
content={"error": str(exc), "code": "JOB_ALREADY_RUNNING"},
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# POST /{job_id}/pause
|
|
109
|
+
@router.post("/{job_id}/pause")
|
|
110
|
+
async def pause_job(job_id: str) -> Any:
|
|
111
|
+
try:
|
|
112
|
+
await scheduler.pause(job_id)
|
|
113
|
+
return {"message": "Job paused"}
|
|
114
|
+
except KeyError:
|
|
115
|
+
return JSONResponse(
|
|
116
|
+
status_code=404,
|
|
117
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
118
|
+
)
|
|
119
|
+
except InvalidOperationError as exc:
|
|
120
|
+
return JSONResponse(
|
|
121
|
+
status_code=409,
|
|
122
|
+
content={"error": str(exc), "code": "JOB_ALREADY_PAUSED"},
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# POST /{job_id}/resume
|
|
126
|
+
@router.post("/{job_id}/resume")
|
|
127
|
+
async def resume_job(job_id: str) -> Any:
|
|
128
|
+
try:
|
|
129
|
+
await scheduler.resume(job_id)
|
|
130
|
+
return {"message": "Job resumed"}
|
|
131
|
+
except KeyError:
|
|
132
|
+
return JSONResponse(
|
|
133
|
+
status_code=404,
|
|
134
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
135
|
+
)
|
|
136
|
+
except InvalidOperationError as exc:
|
|
137
|
+
return JSONResponse(
|
|
138
|
+
status_code=409,
|
|
139
|
+
content={"error": str(exc), "code": "JOB_NOT_PAUSED"},
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# POST /{job_id}/stop
|
|
143
|
+
@router.post("/{job_id}/stop")
|
|
144
|
+
async def stop_job(job_id: str) -> Any:
|
|
145
|
+
try:
|
|
146
|
+
await scheduler.stop_job(job_id)
|
|
147
|
+
return {"message": "Job stopped"}
|
|
148
|
+
except KeyError:
|
|
149
|
+
return JSONResponse(
|
|
150
|
+
status_code=404,
|
|
151
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# POST /{job_id}/cancel
|
|
155
|
+
@router.post("/{job_id}/cancel")
|
|
156
|
+
async def cancel_job(job_id: str) -> Any:
|
|
157
|
+
try:
|
|
158
|
+
await scheduler.cancel_current(job_id)
|
|
159
|
+
return {"message": "Execution cancelled"}
|
|
160
|
+
except KeyError:
|
|
161
|
+
return JSONResponse(
|
|
162
|
+
status_code=404,
|
|
163
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
164
|
+
)
|
|
165
|
+
except InvalidOperationError as exc:
|
|
166
|
+
return JSONResponse(
|
|
167
|
+
status_code=409,
|
|
168
|
+
content={"error": str(exc), "code": "JOB_NOT_RUNNING"},
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
# PUT /{job_id} — update options (patch semantics)
|
|
172
|
+
@router.put("/{job_id}")
|
|
173
|
+
async def update_job(job_id: str, body: UpdateJobRequest) -> Any:
|
|
174
|
+
try:
|
|
175
|
+
await scheduler.update_options(
|
|
176
|
+
job_id,
|
|
177
|
+
description=body.description,
|
|
178
|
+
group=body.group,
|
|
179
|
+
schedule=body.schedule,
|
|
180
|
+
interval=body.interval,
|
|
181
|
+
max_retries=body.max_retries,
|
|
182
|
+
retry_delay=body.retry_delay,
|
|
183
|
+
retry_strategy=body.retry_strategy,
|
|
184
|
+
timeout=body.timeout,
|
|
185
|
+
overlap_policy=body.overlap_policy,
|
|
186
|
+
)
|
|
187
|
+
return {"message": "Job updated"}
|
|
188
|
+
except KeyError:
|
|
189
|
+
return JSONResponse(
|
|
190
|
+
status_code=404,
|
|
191
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
192
|
+
)
|
|
193
|
+
except ValueError as exc:
|
|
194
|
+
return JSONResponse(
|
|
195
|
+
status_code=400,
|
|
196
|
+
content={"error": str(exc), "code": "INVALID_SCHEDULE"},
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
# GET /{job_id}/history
|
|
200
|
+
@router.get("/{job_id}/history")
|
|
201
|
+
async def get_history(
|
|
202
|
+
job_id: str,
|
|
203
|
+
limit: int = Query(20, le=100),
|
|
204
|
+
offset: int = Query(0, ge=0),
|
|
205
|
+
status: str | None = Query(None),
|
|
206
|
+
) -> Any:
|
|
207
|
+
if job_id not in scheduler.registrations:
|
|
208
|
+
return JSONResponse(
|
|
209
|
+
status_code=404,
|
|
210
|
+
content={"error": "Job not found", "code": "JOB_NOT_FOUND"},
|
|
211
|
+
)
|
|
212
|
+
items, total = await storage.get_executions(
|
|
213
|
+
job_id, limit=limit, offset=offset, status=status
|
|
214
|
+
)
|
|
215
|
+
return {
|
|
216
|
+
"items": [
|
|
217
|
+
{
|
|
218
|
+
"id": e.id,
|
|
219
|
+
"jobId": e.job_id,
|
|
220
|
+
"jobName": e.job_name,
|
|
221
|
+
"status": e.status,
|
|
222
|
+
"startedAt": _serialize_dt(e.started_at),
|
|
223
|
+
"completedAt": _serialize_dt(e.completed_at),
|
|
224
|
+
"duration": e.duration,
|
|
225
|
+
"error": e.error,
|
|
226
|
+
"retryAttempt": e.retry_attempt,
|
|
227
|
+
"logCount": len(e.logs),
|
|
228
|
+
}
|
|
229
|
+
for e in items
|
|
230
|
+
],
|
|
231
|
+
"total": total,
|
|
232
|
+
"limit": limit,
|
|
233
|
+
"offset": offset,
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
# GET /{job_id}/history/{exec_id}
|
|
237
|
+
@router.get("/{job_id}/history/{exec_id}")
|
|
238
|
+
async def get_execution(job_id: str, exec_id: str) -> Any:
|
|
239
|
+
execution = await storage.get_execution(exec_id)
|
|
240
|
+
if execution is None or execution.job_id != job_id:
|
|
241
|
+
return JSONResponse(
|
|
242
|
+
status_code=404,
|
|
243
|
+
content={"error": "Execution not found", "code": "EXECUTION_NOT_FOUND"},
|
|
244
|
+
)
|
|
245
|
+
return {
|
|
246
|
+
"id": execution.id,
|
|
247
|
+
"jobId": execution.job_id,
|
|
248
|
+
"jobName": execution.job_name,
|
|
249
|
+
"status": execution.status,
|
|
250
|
+
"startedAt": _serialize_dt(execution.started_at),
|
|
251
|
+
"completedAt": _serialize_dt(execution.completed_at),
|
|
252
|
+
"duration": execution.duration,
|
|
253
|
+
"error": execution.error,
|
|
254
|
+
"retryAttempt": execution.retry_attempt,
|
|
255
|
+
"logs": [
|
|
256
|
+
{
|
|
257
|
+
"timestamp": _serialize_dt(log.timestamp),
|
|
258
|
+
"level": log.level,
|
|
259
|
+
"message": log.message,
|
|
260
|
+
}
|
|
261
|
+
for log in execution.logs
|
|
262
|
+
],
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
return router
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from qodalis_cli_server_abstractions.jobs import (
|
|
4
|
+
ICliJobStorageProvider,
|
|
5
|
+
JobExecution,
|
|
6
|
+
JobState,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class InMemoryJobStorageProvider(ICliJobStorageProvider):
|
|
11
|
+
"""In-memory implementation — data is lost on server restart."""
|
|
12
|
+
|
|
13
|
+
def __init__(self) -> None:
|
|
14
|
+
# job_id -> list of executions (newest first)
|
|
15
|
+
self._executions: dict[str, list[JobExecution]] = {}
|
|
16
|
+
# execution_id -> execution
|
|
17
|
+
self._executions_by_id: dict[str, JobExecution] = {}
|
|
18
|
+
# job_id -> JobState
|
|
19
|
+
self._states: dict[str, JobState] = {}
|
|
20
|
+
|
|
21
|
+
async def save_execution(self, execution: JobExecution) -> None:
|
|
22
|
+
self._executions_by_id[execution.id] = execution
|
|
23
|
+
job_list = self._executions.setdefault(execution.job_id, [])
|
|
24
|
+
# Update if existing, else prepend
|
|
25
|
+
for i, existing in enumerate(job_list):
|
|
26
|
+
if existing.id == execution.id:
|
|
27
|
+
job_list[i] = execution
|
|
28
|
+
return
|
|
29
|
+
job_list.insert(0, execution)
|
|
30
|
+
|
|
31
|
+
async def get_executions(
|
|
32
|
+
self,
|
|
33
|
+
job_id: str,
|
|
34
|
+
*,
|
|
35
|
+
limit: int = 20,
|
|
36
|
+
offset: int = 0,
|
|
37
|
+
status: str | None = None,
|
|
38
|
+
) -> tuple[list[JobExecution], int]:
|
|
39
|
+
all_items = self._executions.get(job_id, [])
|
|
40
|
+
if status:
|
|
41
|
+
all_items = [e for e in all_items if e.status == status]
|
|
42
|
+
total = len(all_items)
|
|
43
|
+
items = all_items[offset : offset + limit]
|
|
44
|
+
return items, total
|
|
45
|
+
|
|
46
|
+
async def get_execution(self, execution_id: str) -> JobExecution | None:
|
|
47
|
+
return self._executions_by_id.get(execution_id)
|
|
48
|
+
|
|
49
|
+
async def save_job_state(self, job_id: str, state: JobState) -> None:
|
|
50
|
+
self._states[job_id] = state
|
|
51
|
+
|
|
52
|
+
async def get_job_state(self, job_id: str) -> JobState | None:
|
|
53
|
+
return self._states.get(job_id)
|
|
54
|
+
|
|
55
|
+
async def get_all_job_states(self) -> dict[str, JobState]:
|
|
56
|
+
return dict(self._states)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
_PATTERN = re.compile(r"^(\d+)(s|m|h|d)$")
|
|
6
|
+
|
|
7
|
+
_MULTIPLIERS = {
|
|
8
|
+
"s": 1.0,
|
|
9
|
+
"m": 60.0,
|
|
10
|
+
"h": 3600.0,
|
|
11
|
+
"d": 86400.0,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def parse_interval(value: str) -> float:
|
|
16
|
+
"""Parse an interval string like ``"30s"``, ``"5m"``, ``"1h"``, ``"1d"``
|
|
17
|
+
and return the number of **seconds** as a float.
|
|
18
|
+
|
|
19
|
+
Raises ``ValueError`` on invalid format.
|
|
20
|
+
"""
|
|
21
|
+
match = _PATTERN.match(value.strip())
|
|
22
|
+
if not match:
|
|
23
|
+
raise ValueError(
|
|
24
|
+
f"Invalid interval format: {value!r}. "
|
|
25
|
+
"Expected a number followed by s, m, h, or d (e.g. '30s', '5m')."
|
|
26
|
+
)
|
|
27
|
+
amount = int(match.group(1))
|
|
28
|
+
unit = match.group(2)
|
|
29
|
+
return amount * _MULTIPLIERS[unit]
|