hexaqueue-server 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hexaqueue_server-0.1.0/PKG-INFO +63 -0
- hexaqueue_server-0.1.0/README.md +44 -0
- hexaqueue_server-0.1.0/pyproject.toml +29 -0
- hexaqueue_server-0.1.0/pyproject.toml.orig +28 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/__init__.py +10 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/adapters/__init__.py +7 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/adapters/local.py +266 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/domain/__init__.py +8 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/domain/models.py +82 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/infra/__init__.py +7 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/infra/bootstrap.py +23 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/ports/__init__.py +7 -0
- hexaqueue_server-0.1.0/src/hexaqueue_server/ports/controller.py +85 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hexaqueue-server
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Hexaqueue Server - High-Availability Central Controller and Scheduler Engine
|
|
5
|
+
Author: Richard West
|
|
6
|
+
Author-email: Richard West <dopplereffect.us@gmail.com>
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Requires-Dist: hexaqueue-core
|
|
9
|
+
Requires-Dist: hexastack-auth>=0.3.0
|
|
10
|
+
Requires-Dist: hexastack-core>=0.3.0
|
|
11
|
+
Requires-Dist: hexastack-cqrs>=0.3.0
|
|
12
|
+
Requires-Dist: hexastack-db>=0.3.0
|
|
13
|
+
Requires-Dist: hexastack-events>=0.3.0
|
|
14
|
+
Requires-Dist: hexastack-fastapi>=0.3.0
|
|
15
|
+
Requires-Dist: hexastack-grpc>=0.3.0
|
|
16
|
+
Requires-Dist: hexastack-otel>=0.3.0
|
|
17
|
+
Requires-Python: >=3.13
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# 🏛️ `hexaqueue-server`
|
|
21
|
+
|
|
22
|
+
> High-Availability Central Controller and Batch Scheduling Daemon.
|
|
23
|
+
|
|
24
|
+
[](https://pypi.org/project/hexaqueue-server/)
|
|
25
|
+
[](https://www.python.org/downloads/)
|
|
26
|
+
[](https://codecov.io/github/TheTrueSCU/hexaqueue)
|
|
27
|
+
[](../../LICENSE)
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## 🏗️ Architecture & Dependencies
|
|
32
|
+
|
|
33
|
+
```mermaid
|
|
34
|
+
graph TD
|
|
35
|
+
subgraph S1["hexaqueue-server"]
|
|
36
|
+
SCHED["Scheduling Engine (Priority Aging, Fair-Share, Backfill, Preemption)"]
|
|
37
|
+
HA["HA Leader Election (Redis / Postgres Lease)"]
|
|
38
|
+
GRPC_API["gRPC Control Plane Service"]
|
|
39
|
+
REST_API["FastAPI REST Management Endpoints"]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
subgraph S2["Internal & Hexastack Dependencies"]
|
|
43
|
+
HQ_CORE["hexaqueue-core"]
|
|
44
|
+
HS_AUTH["hexastack-auth (SPIFFE/mTLS & OIDC)"]
|
|
45
|
+
HS_CORE["hexastack-core"]
|
|
46
|
+
HS_CQRS["hexastack-cqrs"]
|
|
47
|
+
HS_DB["hexastack-db (PostgreSQL / SQLite)"]
|
|
48
|
+
HS_EVENTS["hexastack-events (NATS JetStream & Redis)"]
|
|
49
|
+
HS_FASTAPI["hexastack-fastapi"]
|
|
50
|
+
HS_GRPC["hexastack-grpc"]
|
|
51
|
+
HS_OTEL["hexastack-otel"]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
S1 --> HQ_CORE
|
|
55
|
+
S1 --> HS_AUTH
|
|
56
|
+
S1 --> HS_CORE
|
|
57
|
+
S1 --> HS_CQRS
|
|
58
|
+
S1 --> HS_DB
|
|
59
|
+
S1 --> HS_EVENTS
|
|
60
|
+
S1 --> HS_FASTAPI
|
|
61
|
+
S1 --> HS_GRPC
|
|
62
|
+
S1 --> HS_OTEL
|
|
63
|
+
```
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# 🏛️ `hexaqueue-server`
|
|
2
|
+
|
|
3
|
+
> High-Availability Central Controller and Batch Scheduling Daemon.
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/hexaqueue-server/)
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
7
|
+
[](https://codecov.io/github/TheTrueSCU/hexaqueue)
|
|
8
|
+
[](../../LICENSE)
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 🏗️ Architecture & Dependencies
|
|
13
|
+
|
|
14
|
+
```mermaid
|
|
15
|
+
graph TD
|
|
16
|
+
subgraph S1["hexaqueue-server"]
|
|
17
|
+
SCHED["Scheduling Engine (Priority Aging, Fair-Share, Backfill, Preemption)"]
|
|
18
|
+
HA["HA Leader Election (Redis / Postgres Lease)"]
|
|
19
|
+
GRPC_API["gRPC Control Plane Service"]
|
|
20
|
+
REST_API["FastAPI REST Management Endpoints"]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
subgraph S2["Internal & Hexastack Dependencies"]
|
|
24
|
+
HQ_CORE["hexaqueue-core"]
|
|
25
|
+
HS_AUTH["hexastack-auth (SPIFFE/mTLS & OIDC)"]
|
|
26
|
+
HS_CORE["hexastack-core"]
|
|
27
|
+
HS_CQRS["hexastack-cqrs"]
|
|
28
|
+
HS_DB["hexastack-db (PostgreSQL / SQLite)"]
|
|
29
|
+
HS_EVENTS["hexastack-events (NATS JetStream & Redis)"]
|
|
30
|
+
HS_FASTAPI["hexastack-fastapi"]
|
|
31
|
+
HS_GRPC["hexastack-grpc"]
|
|
32
|
+
HS_OTEL["hexastack-otel"]
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
S1 --> HQ_CORE
|
|
36
|
+
S1 --> HS_AUTH
|
|
37
|
+
S1 --> HS_CORE
|
|
38
|
+
S1 --> HS_CQRS
|
|
39
|
+
S1 --> HS_DB
|
|
40
|
+
S1 --> HS_EVENTS
|
|
41
|
+
S1 --> HS_FASTAPI
|
|
42
|
+
S1 --> HS_GRPC
|
|
43
|
+
S1 --> HS_OTEL
|
|
44
|
+
```
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hexaqueue-server"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Hexaqueue Server - High-Availability Central Controller and Scheduler Engine"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
requires-python = ">=3.13"
|
|
8
|
+
dependencies = [
|
|
9
|
+
"hexaqueue-core",
|
|
10
|
+
"hexastack-auth>=0.3.0",
|
|
11
|
+
"hexastack-core>=0.3.0",
|
|
12
|
+
"hexastack-cqrs>=0.3.0",
|
|
13
|
+
"hexastack-db>=0.3.0",
|
|
14
|
+
"hexastack-events>=0.3.0",
|
|
15
|
+
"hexastack-fastapi>=0.3.0",
|
|
16
|
+
"hexastack-grpc>=0.3.0",
|
|
17
|
+
"hexastack-otel>=0.3.0",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[[project.authors]]
|
|
21
|
+
name = "Richard West"
|
|
22
|
+
email = "dopplereffect.us@gmail.com"
|
|
23
|
+
|
|
24
|
+
[build-system]
|
|
25
|
+
requires = ["uv_build>=0.12.3,<0.13.0"]
|
|
26
|
+
build-backend = "uv_build"
|
|
27
|
+
|
|
28
|
+
[tool.uv.sources.hexaqueue-core]
|
|
29
|
+
workspace = true
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hexaqueue-server"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Hexaqueue Server - High-Availability Central Controller and Scheduler Engine"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "Richard West", email = "dopplereffect.us@gmail.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.13"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"hexaqueue-core",
|
|
13
|
+
"hexastack-auth>=0.3.0",
|
|
14
|
+
"hexastack-core>=0.3.0",
|
|
15
|
+
"hexastack-cqrs>=0.3.0",
|
|
16
|
+
"hexastack-db>=0.3.0",
|
|
17
|
+
"hexastack-events>=0.3.0",
|
|
18
|
+
"hexastack-fastapi>=0.3.0",
|
|
19
|
+
"hexastack-grpc>=0.3.0",
|
|
20
|
+
"hexastack-otel>=0.3.0",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["uv_build>=0.12.3,<0.13.0"]
|
|
25
|
+
build-backend = "uv_build"
|
|
26
|
+
|
|
27
|
+
[tool.uv.sources]
|
|
28
|
+
hexaqueue-core = { workspace = true }
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""Standalone in-process scheduler controller adapter.
|
|
2
|
+
|
|
3
|
+
Notes/Architectural Intent:
|
|
4
|
+
Orchestrates job queues, DAG dependency resolution, and state roll-up
|
|
5
|
+
in-process without requiring external database or cluster daemon overhead.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
|
|
10
|
+
from hexaqueue_core.domain.dag import JobDagEngine, TriggerCondition
|
|
11
|
+
from hexaqueue_core.domain.exceptions import HexaqueueError
|
|
12
|
+
from hexaqueue_core.domain.job import JobSpec
|
|
13
|
+
from hexaqueue_core.domain.lifecycle import (
|
|
14
|
+
JobState,
|
|
15
|
+
JobStatus,
|
|
16
|
+
RunState,
|
|
17
|
+
TerminalOutcome,
|
|
18
|
+
compute_run_outcome,
|
|
19
|
+
compute_run_state,
|
|
20
|
+
)
|
|
21
|
+
from hexaqueue_core.ports.queue import JobQueuePort
|
|
22
|
+
from hexaqueue_server.domain.models import RunStatusReport, RunSubmission
|
|
23
|
+
from hexaqueue_server.ports.controller import SchedulerControllerPort
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LocalSchedulerControllerAdapter(SchedulerControllerPort):
|
|
27
|
+
"""In-process scheduler controller implementation."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, queue: JobQueuePort) -> None:
|
|
30
|
+
"""Initialize controller with task queue.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
queue: Underlying JobQueuePort implementation (e.g. InMemoryJobQueueAdapter).
|
|
34
|
+
"""
|
|
35
|
+
self._queue = queue
|
|
36
|
+
self._runs: dict[str, RunSubmission] = {}
|
|
37
|
+
self._jobs: dict[str, JobSpec] = {}
|
|
38
|
+
self._dag_engines: dict[str, JobDagEngine] = {}
|
|
39
|
+
self._lock = asyncio.Lock()
|
|
40
|
+
|
|
41
|
+
async def submit_run(self, submission: RunSubmission) -> RunStatusReport:
|
|
42
|
+
"""Submit a DAG pipeline run for scheduling."""
|
|
43
|
+
run_id = submission.run_spec.id
|
|
44
|
+
|
|
45
|
+
async with self._lock:
|
|
46
|
+
if run_id in self._runs:
|
|
47
|
+
msg = f"Run with ID '{run_id}' is already registered"
|
|
48
|
+
raise HexaqueueError(msg)
|
|
49
|
+
|
|
50
|
+
# Build and validate DAG
|
|
51
|
+
dag = JobDagEngine()
|
|
52
|
+
for child_id, parents in submission.dependencies.items():
|
|
53
|
+
for parent_id in parents:
|
|
54
|
+
dag.add_dependency(
|
|
55
|
+
child_job_id=child_id,
|
|
56
|
+
parent_job_id=parent_id,
|
|
57
|
+
condition=TriggerCondition.AFTER_OK,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
all_job_ids = {j.id for j in submission.jobs}
|
|
61
|
+
# Throws DependencyCycleError if cyclic
|
|
62
|
+
dag.validate_and_topological_sort(all_job_ids)
|
|
63
|
+
|
|
64
|
+
self._runs[run_id] = submission
|
|
65
|
+
self._dag_engines[run_id] = dag
|
|
66
|
+
|
|
67
|
+
# Register jobs and enqueue root jobs (no prerequisites)
|
|
68
|
+
for job in submission.jobs:
|
|
69
|
+
self._jobs[job.id] = job
|
|
70
|
+
is_ready = dag.is_job_ready(job.id, {})
|
|
71
|
+
if is_ready:
|
|
72
|
+
# Advance to PENDING and enqueue
|
|
73
|
+
pending_job = JobSpec(
|
|
74
|
+
id=job.id,
|
|
75
|
+
run_id=job.run_id,
|
|
76
|
+
name=job.name,
|
|
77
|
+
command=job.command,
|
|
78
|
+
args=job.args,
|
|
79
|
+
env=job.env,
|
|
80
|
+
resources=job.resources,
|
|
81
|
+
collateral_ids=job.collateral_ids,
|
|
82
|
+
tags=job.tags,
|
|
83
|
+
status=JobStatus(state=JobState.PENDING),
|
|
84
|
+
)
|
|
85
|
+
self._jobs[job.id] = pending_job
|
|
86
|
+
await self._queue.enqueue(pending_job)
|
|
87
|
+
|
|
88
|
+
return await self.get_run_status(run_id)
|
|
89
|
+
|
|
90
|
+
async def get_run_status(self, run_id: str) -> RunStatusReport:
|
|
91
|
+
"""Retrieve aggregated run status."""
|
|
92
|
+
async with self._lock:
|
|
93
|
+
if run_id not in self._runs:
|
|
94
|
+
msg = f"Run with ID '{run_id}' not found"
|
|
95
|
+
raise HexaqueueError(msg)
|
|
96
|
+
|
|
97
|
+
submission = self._runs[run_id]
|
|
98
|
+
run_jobs = [self._jobs[j.id] for j in submission.jobs]
|
|
99
|
+
|
|
100
|
+
job_states = [j.state for j in run_jobs]
|
|
101
|
+
run_state = compute_run_state(job_states)
|
|
102
|
+
|
|
103
|
+
job_outcomes = [j.outcome for j in run_jobs if j.outcome is not None]
|
|
104
|
+
run_outcome = (
|
|
105
|
+
compute_run_outcome(job_outcomes)
|
|
106
|
+
if run_state in (RunState.DONE, RunState.BLOCKED)
|
|
107
|
+
else None
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
completed = sum(
|
|
111
|
+
1 for j in run_jobs if j.outcome == TerminalOutcome.COMPLETED
|
|
112
|
+
)
|
|
113
|
+
failed = sum(
|
|
114
|
+
1
|
|
115
|
+
for j in run_jobs
|
|
116
|
+
if j.outcome in (TerminalOutcome.FAILED, TerminalOutcome.TIMED_OUT)
|
|
117
|
+
)
|
|
118
|
+
running = sum(
|
|
119
|
+
1
|
|
120
|
+
for j in run_jobs
|
|
121
|
+
if j.state
|
|
122
|
+
in (JobState.PROVISIONING, JobState.RUNNING, JobState.CLEANUP)
|
|
123
|
+
)
|
|
124
|
+
pending = sum(
|
|
125
|
+
1
|
|
126
|
+
for j in run_jobs
|
|
127
|
+
if j.state in (JobState.SUBMITTED, JobState.BLOCKED, JobState.PENDING)
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
return RunStatusReport(
|
|
131
|
+
run_id=run_id,
|
|
132
|
+
state=run_state,
|
|
133
|
+
outcome=run_outcome,
|
|
134
|
+
total_jobs=len(run_jobs),
|
|
135
|
+
completed_jobs=completed,
|
|
136
|
+
failed_jobs=failed,
|
|
137
|
+
running_jobs=running,
|
|
138
|
+
pending_jobs=pending,
|
|
139
|
+
created_at=submission.run_spec.created_at,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
async def get_job(self, job_id: str) -> JobSpec:
|
|
143
|
+
"""Retrieve current metadata and status for an individual job."""
|
|
144
|
+
async with self._lock:
|
|
145
|
+
if job_id not in self._jobs:
|
|
146
|
+
msg = f"Job with ID '{job_id}' not found"
|
|
147
|
+
raise HexaqueueError(msg)
|
|
148
|
+
return self._jobs[job_id]
|
|
149
|
+
|
|
150
|
+
async def update_job_outcome(
|
|
151
|
+
self,
|
|
152
|
+
job_id: str,
|
|
153
|
+
outcome: TerminalOutcome,
|
|
154
|
+
reason: str | None = None,
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Record terminal outcome and advance eligible downstream dependents."""
|
|
157
|
+
async with self._lock:
|
|
158
|
+
if job_id not in self._jobs:
|
|
159
|
+
msg = f"Job with ID '{job_id}' not found"
|
|
160
|
+
raise HexaqueueError(msg)
|
|
161
|
+
|
|
162
|
+
current_job = self._jobs[job_id]
|
|
163
|
+
run_id = current_job.run_id
|
|
164
|
+
|
|
165
|
+
# Update job to terminal state
|
|
166
|
+
terminal_job = JobSpec(
|
|
167
|
+
id=current_job.id,
|
|
168
|
+
run_id=current_job.run_id,
|
|
169
|
+
name=current_job.name,
|
|
170
|
+
command=current_job.command,
|
|
171
|
+
args=current_job.args,
|
|
172
|
+
env=current_job.env,
|
|
173
|
+
resources=current_job.resources,
|
|
174
|
+
collateral_ids=current_job.collateral_ids,
|
|
175
|
+
tags=current_job.tags,
|
|
176
|
+
status=JobStatus(
|
|
177
|
+
state=JobState.DONE,
|
|
178
|
+
outcome=outcome,
|
|
179
|
+
reason=reason,
|
|
180
|
+
),
|
|
181
|
+
)
|
|
182
|
+
self._jobs[job_id] = terminal_job
|
|
183
|
+
|
|
184
|
+
# Evaluate downstream dependents
|
|
185
|
+
if run_id in self._runs:
|
|
186
|
+
submission = self._runs[run_id]
|
|
187
|
+
dag = self._dag_engines[run_id]
|
|
188
|
+
|
|
189
|
+
# Map current terminal outcomes of run's jobs
|
|
190
|
+
outcomes: dict[str, TerminalOutcome | None] = {
|
|
191
|
+
j.id: self._jobs[j.id].outcome for j in submission.jobs
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
for job_spec in submission.jobs:
|
|
195
|
+
j_id = job_spec.id
|
|
196
|
+
stored = self._jobs[j_id]
|
|
197
|
+
if stored.state == JobState.SUBMITTED:
|
|
198
|
+
if dag.is_job_ready(j_id, outcomes):
|
|
199
|
+
ready_job = JobSpec(
|
|
200
|
+
id=stored.id,
|
|
201
|
+
run_id=stored.run_id,
|
|
202
|
+
name=stored.name,
|
|
203
|
+
command=stored.command,
|
|
204
|
+
args=stored.args,
|
|
205
|
+
env=stored.env,
|
|
206
|
+
resources=stored.resources,
|
|
207
|
+
collateral_ids=stored.collateral_ids,
|
|
208
|
+
tags=stored.tags,
|
|
209
|
+
status=JobStatus(state=JobState.PENDING),
|
|
210
|
+
)
|
|
211
|
+
self._jobs[j_id] = ready_job
|
|
212
|
+
await self._queue.enqueue(ready_job)
|
|
213
|
+
elif dag.is_job_blocked(j_id, outcomes):
|
|
214
|
+
blocked_job = JobSpec(
|
|
215
|
+
id=stored.id,
|
|
216
|
+
run_id=stored.run_id,
|
|
217
|
+
name=stored.name,
|
|
218
|
+
command=stored.command,
|
|
219
|
+
args=stored.args,
|
|
220
|
+
env=stored.env,
|
|
221
|
+
resources=stored.resources,
|
|
222
|
+
collateral_ids=stored.collateral_ids,
|
|
223
|
+
tags=stored.tags,
|
|
224
|
+
status=JobStatus(
|
|
225
|
+
state=JobState.BLOCKED,
|
|
226
|
+
reason="Upstream prerequisite task failed",
|
|
227
|
+
),
|
|
228
|
+
)
|
|
229
|
+
self._jobs[j_id] = blocked_job
|
|
230
|
+
|
|
231
|
+
async def cancel_run(self, run_id: str) -> RunStatusReport:
|
|
232
|
+
"""Cancel run and all active / pending jobs."""
|
|
233
|
+
async with self._lock:
|
|
234
|
+
if run_id not in self._runs:
|
|
235
|
+
msg = f"Run with ID '{run_id}' not found"
|
|
236
|
+
raise HexaqueueError(msg)
|
|
237
|
+
|
|
238
|
+
submission = self._runs[run_id]
|
|
239
|
+
for job in submission.jobs:
|
|
240
|
+
current = self._jobs[job.id]
|
|
241
|
+
if current.state != JobState.DONE:
|
|
242
|
+
await self._queue.remove(job.id)
|
|
243
|
+
cancelled_job = JobSpec(
|
|
244
|
+
id=current.id,
|
|
245
|
+
run_id=current.run_id,
|
|
246
|
+
name=current.name,
|
|
247
|
+
command=current.command,
|
|
248
|
+
args=current.args,
|
|
249
|
+
env=current.env,
|
|
250
|
+
resources=current.resources,
|
|
251
|
+
collateral_ids=current.collateral_ids,
|
|
252
|
+
tags=current.tags,
|
|
253
|
+
status=JobStatus(
|
|
254
|
+
state=JobState.DONE,
|
|
255
|
+
outcome=TerminalOutcome.CANCELLED,
|
|
256
|
+
reason="Run cancelled by user request",
|
|
257
|
+
),
|
|
258
|
+
)
|
|
259
|
+
self._jobs[job.id] = cancelled_job
|
|
260
|
+
|
|
261
|
+
return await self.get_run_status(run_id)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
__all__ = [
|
|
265
|
+
"LocalSchedulerControllerAdapter",
|
|
266
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Domain models for central scheduler controller operations."""
|
|
2
|
+
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Self
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
7
|
+
|
|
8
|
+
from hexaqueue_core.domain.job import JobSpec
|
|
9
|
+
from hexaqueue_core.domain.lifecycle import RunOutcome, RunState
|
|
10
|
+
from hexaqueue_core.domain.run import RunSpec
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RunSubmission(BaseModel):
|
|
14
|
+
"""Submission payload to trigger a multi-job DAG run pipeline.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
run_spec: Root run metadata specification.
|
|
18
|
+
jobs: List of JobSpecs belonging to this pipeline.
|
|
19
|
+
dependencies: Adjacency dictionary mapping child_job_id -> list of prerequisite parent_job_ids.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
23
|
+
|
|
24
|
+
dependencies: dict[str, list[str]] = Field(
|
|
25
|
+
default_factory=dict, description="Job dependency adjacency mapping"
|
|
26
|
+
)
|
|
27
|
+
jobs: list[JobSpec] = Field(min_length=1, description="List of jobs in this run")
|
|
28
|
+
run_spec: RunSpec = Field(description="Pipeline run specification")
|
|
29
|
+
|
|
30
|
+
@model_validator(mode="after")
|
|
31
|
+
def validate_invariants(self) -> Self:
|
|
32
|
+
"""Validate submission invariants."""
|
|
33
|
+
job_ids = {j.id for j in self.jobs}
|
|
34
|
+
for child_id, parents in self.dependencies.items():
|
|
35
|
+
if child_id not in job_ids:
|
|
36
|
+
msg = f"Dependency child job '{child_id}' not found in jobs list"
|
|
37
|
+
raise ValueError(msg)
|
|
38
|
+
for parent_id in parents:
|
|
39
|
+
if parent_id not in job_ids:
|
|
40
|
+
msg = f"Dependency parent job '{parent_id}' not found in jobs list"
|
|
41
|
+
raise ValueError(msg)
|
|
42
|
+
return self
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class RunStatusReport(BaseModel):
|
|
46
|
+
"""Comprehensive snapshot of a pipeline run's execution status.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
run_id: Root run identifier.
|
|
50
|
+
state: Aggregated RunState (PENDING, RUNNING, DONE).
|
|
51
|
+
outcome: Terminal RunOutcome (COMPLETED, FAILED, TIMED_OUT, CANCELLED) if DONE.
|
|
52
|
+
total_jobs: Total jobs count in pipeline.
|
|
53
|
+
completed_jobs: Number of completed jobs.
|
|
54
|
+
failed_jobs: Number of failed jobs.
|
|
55
|
+
running_jobs: Number of currently executing jobs.
|
|
56
|
+
pending_jobs: Number of pending/blocked jobs.
|
|
57
|
+
created_at: Run creation timestamp.
|
|
58
|
+
updated_at: Status report generation timestamp.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
62
|
+
|
|
63
|
+
completed_jobs: int = Field(ge=0, description="Completed jobs count")
|
|
64
|
+
created_at: datetime = Field(description="Run creation timestamp")
|
|
65
|
+
failed_jobs: int = Field(ge=0, description="Failed jobs count")
|
|
66
|
+
outcome: RunOutcome | None = Field(
|
|
67
|
+
default=None, description="Terminal outcome if DONE"
|
|
68
|
+
)
|
|
69
|
+
pending_jobs: int = Field(ge=0, description="Pending jobs count")
|
|
70
|
+
run_id: str = Field(description="Run identifier")
|
|
71
|
+
running_jobs: int = Field(ge=0, description="Running jobs count")
|
|
72
|
+
state: RunState = Field(description="Aggregate run state")
|
|
73
|
+
total_jobs: int = Field(ge=1, description="Total jobs count")
|
|
74
|
+
updated_at: datetime = Field(
|
|
75
|
+
default_factory=lambda: datetime.now(UTC), description="Update timestamp"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
__all__ = [
|
|
80
|
+
"RunStatusReport",
|
|
81
|
+
"RunSubmission",
|
|
82
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Bootstrap lifecycle handler for hexaqueue-server."""
|
|
2
|
+
|
|
3
|
+
from hexaqueue_core.infra.bootstrap import HexaqueueBootstrapContext
|
|
4
|
+
from hexaqueue_core.infra.registries import HexaqueueConfigRegistry
|
|
5
|
+
from hexaqueue_core.ports.bootstrap import BootstrapperPort
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ServerBootstrapper(BootstrapperPort):
|
|
9
|
+
"""Bootstrapper for hexaqueue-server services and registries."""
|
|
10
|
+
|
|
11
|
+
name: str = "server"
|
|
12
|
+
order: int = 30
|
|
13
|
+
|
|
14
|
+
def register_config(self, registry: HexaqueueConfigRegistry) -> None:
|
|
15
|
+
"""Register server configuration models."""
|
|
16
|
+
|
|
17
|
+
def configure(self, context: HexaqueueBootstrapContext) -> None:
|
|
18
|
+
"""Configure server services in application DI container."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"ServerBootstrapper",
|
|
23
|
+
]
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Scheduler controller port interface."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from hexaqueue_core.domain.job import JobSpec
|
|
6
|
+
from hexaqueue_core.domain.lifecycle import TerminalOutcome
|
|
7
|
+
from hexaqueue_server.domain.models import RunStatusReport, RunSubmission
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SchedulerControllerPort(ABC):
|
|
11
|
+
"""Abstract port for pipeline submission, scheduling coordination, and status reporting."""
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
async def submit_run(self, submission: RunSubmission) -> RunStatusReport:
|
|
15
|
+
"""Submit a DAG pipeline run for scheduling.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
submission: Complete RunSubmission specification.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
Initial RunStatusReport.
|
|
22
|
+
|
|
23
|
+
Raises:
|
|
24
|
+
DependencyCycleError: If dependency graph contains cycles.
|
|
25
|
+
HexaqueueError: If submission fails validation.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
async def get_run_status(self, run_id: str) -> RunStatusReport:
|
|
30
|
+
"""Retrieve aggregated run status.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
run_id: Unique pipeline run identifier.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
Current RunStatusReport.
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
HexaqueueError: If run is not found.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
async def get_job(self, job_id: str) -> JobSpec:
|
|
44
|
+
"""Retrieve current metadata and status for an individual job.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
job_id: Unique job identifier.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
JobSpec instance.
|
|
51
|
+
|
|
52
|
+
Raises:
|
|
53
|
+
HexaqueueError: If job is not found.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
@abstractmethod
|
|
57
|
+
async def update_job_outcome(
|
|
58
|
+
self,
|
|
59
|
+
job_id: str,
|
|
60
|
+
outcome: TerminalOutcome,
|
|
61
|
+
reason: str | None = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Record the terminal outcome of an executed job and advance dependent downstream tasks.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
job_id: Finished job identifier.
|
|
67
|
+
outcome: Final TerminalOutcome (COMPLETED, FAILED, TIMED_OUT, CANCELLED).
|
|
68
|
+
reason: Optional explanation or error message.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
@abstractmethod
|
|
72
|
+
async def cancel_run(self, run_id: str) -> RunStatusReport:
|
|
73
|
+
"""Cancel an in-flight run and all of its remaining non-terminal jobs.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
run_id: Pipeline run identifier to cancel.
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
Updated RunStatusReport showing CANCELLED state.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
__all__ = [
|
|
84
|
+
"SchedulerControllerPort",
|
|
85
|
+
]
|