rampgate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rampgate/__init__.py +234 -0
- rampgate/actions.py +316 -0
- rampgate/config.py +101 -0
- rampgate/decide.py +652 -0
- rampgate/engine.py +322 -0
- rampgate/estimators.py +406 -0
- rampgate/events.py +190 -0
- rampgate/explain.py +126 -0
- rampgate/model.py +363 -0
- rampgate/policies/__init__.py +16 -0
- rampgate/policies/base.py +57 -0
- rampgate/policies/fifo.py +22 -0
- rampgate/policies/priority.py +34 -0
- rampgate/policies/stage.py +132 -0
- rampgate/py.typed +0 -0
- rampgate/reducer.py +832 -0
- rampgate/resources.py +192 -0
- rampgate/serialization.py +225 -0
- rampgate/sim/__init__.py +63 -0
- rampgate/sim/baselines.py +187 -0
- rampgate/sim/clock.py +21 -0
- rampgate/sim/executor.py +303 -0
- rampgate/sim/report.py +281 -0
- rampgate/sim/scenario.py +297 -0
- rampgate/sim/workloads.py +121 -0
- rampgate/stability.py +77 -0
- rampgate-0.1.0.dist-info/METADATA +58 -0
- rampgate-0.1.0.dist-info/RECORD +31 -0
- rampgate-0.1.0.dist-info/WHEEL +4 -0
- rampgate-0.1.0.dist-info/licenses/LICENSE +202 -0
- rampgate-0.1.0.dist-info/licenses/NOTICE +4 -0
rampgate/__init__.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""rampgate: observe-then-reserve admission scheduler."""
|
|
2
|
+
|
|
3
|
+
from rampgate.actions import (
|
|
4
|
+
Action,
|
|
5
|
+
CapacityExceeded,
|
|
6
|
+
Fail,
|
|
7
|
+
FailReason,
|
|
8
|
+
ForeignUsageHigh,
|
|
9
|
+
GroupCompleted,
|
|
10
|
+
GroupOutcome,
|
|
11
|
+
HintExceeded,
|
|
12
|
+
JobNotObserved,
|
|
13
|
+
MarkStable,
|
|
14
|
+
NoUsageObserved,
|
|
15
|
+
Overcommitted,
|
|
16
|
+
QueueStopped,
|
|
17
|
+
Requeue,
|
|
18
|
+
RequeueReason,
|
|
19
|
+
Start,
|
|
20
|
+
Stop,
|
|
21
|
+
UncleanStop,
|
|
22
|
+
Unreconciled,
|
|
23
|
+
UpdateReservation,
|
|
24
|
+
Warn,
|
|
25
|
+
WarnCleared,
|
|
26
|
+
WarningKind,
|
|
27
|
+
is_edge_triggered,
|
|
28
|
+
key_id,
|
|
29
|
+
warning_key,
|
|
30
|
+
)
|
|
31
|
+
from rampgate.config import Config
|
|
32
|
+
from rampgate.decide import (
|
|
33
|
+
PROBATION,
|
|
34
|
+
Admission,
|
|
35
|
+
Decision,
|
|
36
|
+
DeviceLoad,
|
|
37
|
+
ProbationAdmission,
|
|
38
|
+
decide,
|
|
39
|
+
device_loads,
|
|
40
|
+
measured_free,
|
|
41
|
+
)
|
|
42
|
+
from rampgate.engine import Engine, JobSample
|
|
43
|
+
from rampgate.estimators import (
|
|
44
|
+
EstimateResult,
|
|
45
|
+
Estimator,
|
|
46
|
+
HistoryEntry,
|
|
47
|
+
HistoryEstimator,
|
|
48
|
+
HistoryStore,
|
|
49
|
+
InMemoryHistoryStore,
|
|
50
|
+
NullEstimator,
|
|
51
|
+
StaticEstimator,
|
|
52
|
+
apply_history,
|
|
53
|
+
resolve_estimates,
|
|
54
|
+
)
|
|
55
|
+
from rampgate.events import (
|
|
56
|
+
UNSET,
|
|
57
|
+
CancelRequested,
|
|
58
|
+
DeviceAdded,
|
|
59
|
+
DeviceObserved,
|
|
60
|
+
DeviceRemoved,
|
|
61
|
+
Event,
|
|
62
|
+
JobExited,
|
|
63
|
+
JobObserved,
|
|
64
|
+
JobStarted,
|
|
65
|
+
JobSubmitted,
|
|
66
|
+
JobUpdated,
|
|
67
|
+
Paused,
|
|
68
|
+
PreemptRequested,
|
|
69
|
+
ResumeRequested,
|
|
70
|
+
Unpaused,
|
|
71
|
+
)
|
|
72
|
+
from rampgate.explain import (
|
|
73
|
+
AwaitingResume,
|
|
74
|
+
BlockedByForeignUsage,
|
|
75
|
+
DeviceHeldFor,
|
|
76
|
+
EnginePaused,
|
|
77
|
+
Explanation,
|
|
78
|
+
GangPartiallyBlocked,
|
|
79
|
+
GroupBarrier,
|
|
80
|
+
InsufficientResources,
|
|
81
|
+
RejectedByAdmission,
|
|
82
|
+
UnknownEstimateBlocked,
|
|
83
|
+
WaitingForIsolation,
|
|
84
|
+
WaitingForWarmup,
|
|
85
|
+
)
|
|
86
|
+
from rampgate.model import (
|
|
87
|
+
ACTIVE_STATES,
|
|
88
|
+
TERMINAL_STATES,
|
|
89
|
+
Device,
|
|
90
|
+
DeviceId,
|
|
91
|
+
DeviceState,
|
|
92
|
+
ExitKind,
|
|
93
|
+
GroupId,
|
|
94
|
+
HistoryRecord,
|
|
95
|
+
Job,
|
|
96
|
+
JobId,
|
|
97
|
+
JobRuntime,
|
|
98
|
+
JobState,
|
|
99
|
+
Snapshot,
|
|
100
|
+
State,
|
|
101
|
+
StopReason,
|
|
102
|
+
)
|
|
103
|
+
from rampgate.policies import (
|
|
104
|
+
BasePolicy,
|
|
105
|
+
FailurePolicy,
|
|
106
|
+
FifoPolicy,
|
|
107
|
+
OrderingPolicy,
|
|
108
|
+
PriorityPolicy,
|
|
109
|
+
StagePolicy,
|
|
110
|
+
StateView,
|
|
111
|
+
)
|
|
112
|
+
from rampgate.reducer import IgnoredEvent, IgnoreReason, InvalidTransition, Message, drain, reduce
|
|
113
|
+
from rampgate.resources import GiB, KiB, MiB, Number, Resources, TiB
|
|
114
|
+
from rampgate.serialization import UnsupportedSchema, dump_state, load_state
|
|
115
|
+
from rampgate.stability import is_peak_growth, is_stable
|
|
116
|
+
|
|
117
|
+
__version__ = "0.1.0"
|
|
118
|
+
|
|
119
|
+
__all__ = [
|
|
120
|
+
"ACTIVE_STATES",
|
|
121
|
+
"PROBATION",
|
|
122
|
+
"TERMINAL_STATES",
|
|
123
|
+
"UNSET",
|
|
124
|
+
"Action",
|
|
125
|
+
"Admission",
|
|
126
|
+
"AwaitingResume",
|
|
127
|
+
"BasePolicy",
|
|
128
|
+
"BlockedByForeignUsage",
|
|
129
|
+
"CancelRequested",
|
|
130
|
+
"CapacityExceeded",
|
|
131
|
+
"Config",
|
|
132
|
+
"Decision",
|
|
133
|
+
"Device",
|
|
134
|
+
"DeviceAdded",
|
|
135
|
+
"DeviceHeldFor",
|
|
136
|
+
"DeviceId",
|
|
137
|
+
"DeviceLoad",
|
|
138
|
+
"DeviceObserved",
|
|
139
|
+
"DeviceRemoved",
|
|
140
|
+
"DeviceState",
|
|
141
|
+
"Engine",
|
|
142
|
+
"EnginePaused",
|
|
143
|
+
"EstimateResult",
|
|
144
|
+
"Estimator",
|
|
145
|
+
"Event",
|
|
146
|
+
"ExitKind",
|
|
147
|
+
"Explanation",
|
|
148
|
+
"Fail",
|
|
149
|
+
"FailReason",
|
|
150
|
+
"FailurePolicy",
|
|
151
|
+
"FifoPolicy",
|
|
152
|
+
"ForeignUsageHigh",
|
|
153
|
+
"GangPartiallyBlocked",
|
|
154
|
+
"GiB",
|
|
155
|
+
"GroupBarrier",
|
|
156
|
+
"GroupCompleted",
|
|
157
|
+
"GroupId",
|
|
158
|
+
"GroupOutcome",
|
|
159
|
+
"HintExceeded",
|
|
160
|
+
"HistoryEntry",
|
|
161
|
+
"HistoryEstimator",
|
|
162
|
+
"HistoryRecord",
|
|
163
|
+
"HistoryStore",
|
|
164
|
+
"IgnoreReason",
|
|
165
|
+
"IgnoredEvent",
|
|
166
|
+
"InMemoryHistoryStore",
|
|
167
|
+
"InsufficientResources",
|
|
168
|
+
"InvalidTransition",
|
|
169
|
+
"Job",
|
|
170
|
+
"JobExited",
|
|
171
|
+
"JobId",
|
|
172
|
+
"JobNotObserved",
|
|
173
|
+
"JobObserved",
|
|
174
|
+
"JobRuntime",
|
|
175
|
+
"JobSample",
|
|
176
|
+
"JobStarted",
|
|
177
|
+
"JobState",
|
|
178
|
+
"JobSubmitted",
|
|
179
|
+
"JobUpdated",
|
|
180
|
+
"KiB",
|
|
181
|
+
"MarkStable",
|
|
182
|
+
"Message",
|
|
183
|
+
"MiB",
|
|
184
|
+
"NoUsageObserved",
|
|
185
|
+
"NullEstimator",
|
|
186
|
+
"Number",
|
|
187
|
+
"OrderingPolicy",
|
|
188
|
+
"Overcommitted",
|
|
189
|
+
"Paused",
|
|
190
|
+
"PreemptRequested",
|
|
191
|
+
"PriorityPolicy",
|
|
192
|
+
"ProbationAdmission",
|
|
193
|
+
"QueueStopped",
|
|
194
|
+
"RejectedByAdmission",
|
|
195
|
+
"Requeue",
|
|
196
|
+
"RequeueReason",
|
|
197
|
+
"Resources",
|
|
198
|
+
"ResumeRequested",
|
|
199
|
+
"Snapshot",
|
|
200
|
+
"StagePolicy",
|
|
201
|
+
"Start",
|
|
202
|
+
"State",
|
|
203
|
+
"StateView",
|
|
204
|
+
"StaticEstimator",
|
|
205
|
+
"Stop",
|
|
206
|
+
"StopReason",
|
|
207
|
+
"TiB",
|
|
208
|
+
"UncleanStop",
|
|
209
|
+
"UnknownEstimateBlocked",
|
|
210
|
+
"Unpaused",
|
|
211
|
+
"Unreconciled",
|
|
212
|
+
"UnsupportedSchema",
|
|
213
|
+
"UpdateReservation",
|
|
214
|
+
"WaitingForIsolation",
|
|
215
|
+
"WaitingForWarmup",
|
|
216
|
+
"Warn",
|
|
217
|
+
"WarnCleared",
|
|
218
|
+
"WarningKind",
|
|
219
|
+
"__version__",
|
|
220
|
+
"apply_history",
|
|
221
|
+
"decide",
|
|
222
|
+
"device_loads",
|
|
223
|
+
"drain",
|
|
224
|
+
"dump_state",
|
|
225
|
+
"is_edge_triggered",
|
|
226
|
+
"is_peak_growth",
|
|
227
|
+
"is_stable",
|
|
228
|
+
"key_id",
|
|
229
|
+
"load_state",
|
|
230
|
+
"measured_free",
|
|
231
|
+
"reduce",
|
|
232
|
+
"resolve_estimates",
|
|
233
|
+
"warning_key",
|
|
234
|
+
]
|
rampgate/actions.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""Output actions (docs/concepts.md), each with a deterministic ``action_id``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import dataclasses
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from enum import Enum, StrEnum
|
|
11
|
+
from typing import Any, TypeAlias
|
|
12
|
+
|
|
13
|
+
from rampgate.model import DeviceId, GroupId, JobId, StopReason, frozen_map
|
|
14
|
+
from rampgate.resources import Resources
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Action",
|
|
18
|
+
"CapacityExceeded",
|
|
19
|
+
"Fail",
|
|
20
|
+
"FailReason",
|
|
21
|
+
"ForeignUsageHigh",
|
|
22
|
+
"GroupCompleted",
|
|
23
|
+
"GroupOutcome",
|
|
24
|
+
"HintExceeded",
|
|
25
|
+
"JobNotObserved",
|
|
26
|
+
"MarkStable",
|
|
27
|
+
"NoUsageObserved",
|
|
28
|
+
"Overcommitted",
|
|
29
|
+
"QueueStopped",
|
|
30
|
+
"Requeue",
|
|
31
|
+
"RequeueReason",
|
|
32
|
+
"Start",
|
|
33
|
+
"Stop",
|
|
34
|
+
"UncleanStop",
|
|
35
|
+
"Unreconciled",
|
|
36
|
+
"UpdateReservation",
|
|
37
|
+
"Warn",
|
|
38
|
+
"WarnCleared",
|
|
39
|
+
"WarningKind",
|
|
40
|
+
"canonical",
|
|
41
|
+
"is_edge_triggered",
|
|
42
|
+
"key_id",
|
|
43
|
+
"warning_key",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class RequeueReason(StrEnum):
|
|
48
|
+
OOM = "OOM"
|
|
49
|
+
START_TIMEOUT = "START_TIMEOUT"
|
|
50
|
+
LOST = "LOST"
|
|
51
|
+
VACATED = "VACATED" # Rule V: stopped to relieve an overloaded device, back to PENDING
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FailReason(StrEnum):
|
|
55
|
+
OOM_RETRIES_EXHAUSTED = "OOM_RETRIES_EXHAUSTED"
|
|
56
|
+
LOST_RETRIES_EXHAUSTED = "LOST_RETRIES_EXHAUSTED"
|
|
57
|
+
EXCEEDS_DEVICE_CAPACITY = "EXCEEDS_DEVICE_CAPACITY"
|
|
58
|
+
EXITED_FAILURE = "EXITED_FAILURE"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class GroupOutcome(StrEnum):
|
|
62
|
+
OK = "OK"
|
|
63
|
+
HAD_FAILURES = "HAD_FAILURES"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def canonical(value: Any) -> Any:
|
|
67
|
+
"""Convert a value made of dataclasses, enums, mappings and Resources into plain JSON data."""
|
|
68
|
+
if isinstance(value, Resources):
|
|
69
|
+
return {"__resources__": value.to_json()}
|
|
70
|
+
if isinstance(value, Enum):
|
|
71
|
+
return value.value
|
|
72
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
73
|
+
return {
|
|
74
|
+
"__type__": type(value).__name__,
|
|
75
|
+
**{f.name: canonical(getattr(value, f.name)) for f in dataclasses.fields(value)},
|
|
76
|
+
}
|
|
77
|
+
if isinstance(value, Mapping):
|
|
78
|
+
return {str(k): canonical(v) for k, v in sorted(value.items())}
|
|
79
|
+
if isinstance(value, tuple | list):
|
|
80
|
+
return [canonical(v) for v in value]
|
|
81
|
+
if isinstance(value, frozenset):
|
|
82
|
+
return sorted(canonical(v) for v in value)
|
|
83
|
+
if value is None or isinstance(value, bool | int | float | str):
|
|
84
|
+
return value
|
|
85
|
+
raise TypeError(f"cannot canonicalize {type(value).__name__}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class _ActionMixin:
|
|
89
|
+
__slots__ = ()
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def action_id(self) -> str:
|
|
93
|
+
"""Hash of the action's type and content (which includes the attempt, where relevant)."""
|
|
94
|
+
blob = json.dumps(canonical(self), sort_keys=True, separators=(",", ":"))
|
|
95
|
+
return hashlib.sha256(blob.encode()).hexdigest()[:32]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True, slots=True)
|
|
99
|
+
class Start(_ActionMixin):
|
|
100
|
+
job_id: JobId
|
|
101
|
+
attempt: int
|
|
102
|
+
placement: tuple[DeviceId, ...]
|
|
103
|
+
resume: bool
|
|
104
|
+
admitted_free: Mapping[DeviceId, Resources]
|
|
105
|
+
issued_at: float
|
|
106
|
+
estimate: Mapping[DeviceId, Resources | None] = field(default_factory=frozen_map)
|
|
107
|
+
trusted: frozenset[DeviceId] = frozenset()
|
|
108
|
+
# Per trusted device: complete observations behind the estimate (rule A, docs/algorithm.md).
|
|
109
|
+
observations: Mapping[DeviceId, int] = field(default_factory=frozen_map)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(frozen=True, slots=True)
|
|
113
|
+
class Stop(_ActionMixin):
|
|
114
|
+
job_id: JobId
|
|
115
|
+
attempt: int
|
|
116
|
+
reason: StopReason
|
|
117
|
+
grace_s: float
|
|
118
|
+
# Rule V (docs/algorithm.md): stopped by rampgate itself; the job returns to PENDING after exit.
|
|
119
|
+
vacate: bool = False
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True, slots=True)
|
|
123
|
+
class MarkStable(_ActionMixin):
|
|
124
|
+
job_id: JobId
|
|
125
|
+
attempt: int
|
|
126
|
+
reservation: Mapping[DeviceId, Resources]
|
|
127
|
+
issued_at: float | None = None # decide's ``now``; the probation slot is kept until then
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True, slots=True)
|
|
131
|
+
class UpdateReservation(_ActionMixin):
|
|
132
|
+
job_id: JobId
|
|
133
|
+
attempt: int
|
|
134
|
+
reservation: Mapping[DeviceId, Resources]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@dataclass(frozen=True, slots=True)
|
|
138
|
+
class Requeue(_ActionMixin):
|
|
139
|
+
job_id: JobId
|
|
140
|
+
next_attempt: int
|
|
141
|
+
new_estimate: Mapping[DeviceId, Resources | None]
|
|
142
|
+
reason: RequeueReason
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(frozen=True, slots=True)
|
|
146
|
+
class Fail(_ActionMixin):
|
|
147
|
+
job_id: JobId
|
|
148
|
+
attempt: int
|
|
149
|
+
reason: FailReason
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@dataclass(frozen=True, slots=True)
|
|
153
|
+
class GroupCompleted(_ActionMixin):
|
|
154
|
+
group: GroupId
|
|
155
|
+
outcome: GroupOutcome
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@dataclass(frozen=True, slots=True)
|
|
159
|
+
class CapacityExceeded:
|
|
160
|
+
"""A candidate's estimate exceeds the device's capacity: it can never fit there."""
|
|
161
|
+
|
|
162
|
+
group: GroupId | None
|
|
163
|
+
device_id: DeviceId
|
|
164
|
+
estimated: Resources
|
|
165
|
+
available: Resources
|
|
166
|
+
job_id: JobId
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@dataclass(frozen=True, slots=True)
|
|
170
|
+
class HintExceeded:
|
|
171
|
+
"""A running job's observed peak exceeds its declared hint (informational, per attempt)."""
|
|
172
|
+
|
|
173
|
+
job_id: JobId
|
|
174
|
+
attempt: int
|
|
175
|
+
device_id: DeviceId
|
|
176
|
+
hint: Resources
|
|
177
|
+
observed: Resources
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass(frozen=True, slots=True)
|
|
181
|
+
class NoUsageObserved:
|
|
182
|
+
"""A WARMING job that reports no steps reached ``max_warmup_s`` without its usage on
|
|
183
|
+
the device ever exceeding the device's ``stable_min_usage``: Rule S keeps it WARMING."""
|
|
184
|
+
|
|
185
|
+
job_id: JobId
|
|
186
|
+
attempt: int
|
|
187
|
+
device_id: DeviceId
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass(frozen=True, slots=True)
|
|
191
|
+
class JobNotObserved:
|
|
192
|
+
"""A running job has not been observed on a device for ``missing_samples_warn``
|
|
193
|
+
consecutive device samples (its last known usage is still used)."""
|
|
194
|
+
|
|
195
|
+
job_id: JobId
|
|
196
|
+
attempt: int
|
|
197
|
+
device_id: DeviceId
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@dataclass(frozen=True, slots=True)
|
|
201
|
+
class Overcommitted:
|
|
202
|
+
"""``sum(R) > C - F`` on a device; ``capacity`` is ``C - F``."""
|
|
203
|
+
|
|
204
|
+
device_id: DeviceId
|
|
205
|
+
reserved: Resources
|
|
206
|
+
capacity: Resources
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@dataclass(frozen=True, slots=True)
|
|
210
|
+
class ForeignUsageHigh:
|
|
211
|
+
"""Foreign usage keeps candidates from fitting on a device; they are listed when raised."""
|
|
212
|
+
|
|
213
|
+
device_id: DeviceId
|
|
214
|
+
foreign: Resources
|
|
215
|
+
blocked_job_ids: tuple[JobId, ...]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass(frozen=True, slots=True)
|
|
219
|
+
class UncleanStop:
|
|
220
|
+
"""A stopped job exited non-zero: work since its last checkpoint may be lost."""
|
|
221
|
+
|
|
222
|
+
job_id: JobId
|
|
223
|
+
attempt: int
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
@dataclass(frozen=True, slots=True)
|
|
227
|
+
class Unreconciled:
|
|
228
|
+
"""After a restart, nobody reported on this running job within ``reconcile_grace_s``."""
|
|
229
|
+
|
|
230
|
+
job_id: JobId
|
|
231
|
+
attempt: int
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@dataclass(frozen=True, slots=True)
|
|
235
|
+
class QueueStopped:
|
|
236
|
+
"""A job of a STOP_QUEUE group failed; reducing this warning pauses the engine."""
|
|
237
|
+
|
|
238
|
+
group: GroupId
|
|
239
|
+
job_id: JobId
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
WarningKind: TypeAlias = (
|
|
243
|
+
CapacityExceeded
|
|
244
|
+
| Overcommitted
|
|
245
|
+
| ForeignUsageHigh
|
|
246
|
+
| UncleanStop
|
|
247
|
+
| QueueStopped
|
|
248
|
+
| Unreconciled
|
|
249
|
+
| HintExceeded
|
|
250
|
+
| NoUsageObserved
|
|
251
|
+
| JobNotObserved
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
EDGE_TRIGGERED: tuple[type, ...] = (
|
|
256
|
+
CapacityExceeded,
|
|
257
|
+
Overcommitted,
|
|
258
|
+
ForeignUsageHigh,
|
|
259
|
+
Unreconciled,
|
|
260
|
+
HintExceeded,
|
|
261
|
+
NoUsageObserved,
|
|
262
|
+
JobNotObserved,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def warning_key(w: WarningKind) -> tuple[str, tuple[str, ...]]:
|
|
267
|
+
"""``(kind, key)`` of a warning (docs/concepts.md warning table)."""
|
|
268
|
+
kind = type(w).__name__
|
|
269
|
+
match w:
|
|
270
|
+
case CapacityExceeded():
|
|
271
|
+
return kind, (w.job_id, w.device_id)
|
|
272
|
+
case Overcommitted() | ForeignUsageHigh():
|
|
273
|
+
return kind, (w.device_id,)
|
|
274
|
+
case Unreconciled():
|
|
275
|
+
return kind, (w.job_id,)
|
|
276
|
+
case UncleanStop():
|
|
277
|
+
return kind, (w.job_id, str(w.attempt))
|
|
278
|
+
case HintExceeded() | NoUsageObserved() | JobNotObserved():
|
|
279
|
+
return kind, (w.job_id, str(w.attempt), w.device_id)
|
|
280
|
+
case QueueStopped():
|
|
281
|
+
return kind, (w.group, w.job_id)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def key_id(kind: str, key: tuple[str, ...]) -> str:
|
|
285
|
+
"""A single string for a ``(kind, key)`` pair, used as a mapping key in the state."""
|
|
286
|
+
return json.dumps([kind, *key], separators=(",", ":"))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def is_edge_triggered(w: WarningKind) -> bool:
|
|
290
|
+
return isinstance(w, EDGE_TRIGGERED)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@dataclass(frozen=True, slots=True)
|
|
294
|
+
class Warn(_ActionMixin):
|
|
295
|
+
warning: WarningKind
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True, slots=True)
|
|
299
|
+
class WarnCleared(_ActionMixin):
|
|
300
|
+
"""The condition of an edge-triggered warning no longer holds."""
|
|
301
|
+
|
|
302
|
+
kind: str
|
|
303
|
+
key: tuple[str, ...]
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
Action: TypeAlias = (
|
|
307
|
+
Start
|
|
308
|
+
| Stop
|
|
309
|
+
| MarkStable
|
|
310
|
+
| UpdateReservation
|
|
311
|
+
| Requeue
|
|
312
|
+
| Fail
|
|
313
|
+
| GroupCompleted
|
|
314
|
+
| Warn
|
|
315
|
+
| WarnCleared
|
|
316
|
+
)
|
rampgate/config.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""`Config`: every tunable threshold of the algorithm (docs/algorithm.md)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from dataclasses import dataclass, fields
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
from rampgate.resources import Resources
|
|
10
|
+
|
|
11
|
+
__all__ = ["Config"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
15
|
+
class Config:
|
|
16
|
+
"""Algorithm thresholds. Times are in seconds of the caller's clock."""
|
|
17
|
+
|
|
18
|
+
# Reservations based on one observation (a STABLE job's peak).
|
|
19
|
+
safety_margin: float = 1.15
|
|
20
|
+
# A trusted STARTING/WARMING job: its estimate is already a max of observations.
|
|
21
|
+
trusted_margin: float = 1.05
|
|
22
|
+
# ...but only with this many complete observations of its level-0 key (rule A).
|
|
23
|
+
min_trusted_observations: int = 3
|
|
24
|
+
min_warmup_s: float = 30
|
|
25
|
+
stable_window_s: float = 60
|
|
26
|
+
stable_min_steps: int = 20
|
|
27
|
+
stable_requires_epoch: bool = True
|
|
28
|
+
growth_tol: float = 0.02
|
|
29
|
+
max_warmup_s: float = 600
|
|
30
|
+
oom_growth: float = 1.25
|
|
31
|
+
max_oom_retries: int = 2
|
|
32
|
+
max_lost_retries: int = 3
|
|
33
|
+
preempt_grace_s: float = 120
|
|
34
|
+
foreign_window_s: float = 10
|
|
35
|
+
foreign_tolerance: float = 0.02
|
|
36
|
+
start_timeout_s: float = 300
|
|
37
|
+
reconcile_grace_s: float = 60
|
|
38
|
+
unknown_min_free: Resources | float = 0.10
|
|
39
|
+
# Rule P for jobs without a trusted estimate only (default), or for every job.
|
|
40
|
+
probation: Literal["untrusted_only", "all"] = "untrusted_only"
|
|
41
|
+
# Rule V: vacate when managed load exceeds C * (1 - headroom). From the simulator's
|
|
42
|
+
# noise bound (2 %); calibrate on real hardware.
|
|
43
|
+
vacate_headroom: float = 0.02
|
|
44
|
+
# Warn(JobNotObserved) after this many consecutive device samples without an
|
|
45
|
+
# observation of a running job there (0: never).
|
|
46
|
+
missing_samples_warn: int = 5
|
|
47
|
+
|
|
48
|
+
def __post_init__(self) -> None:
|
|
49
|
+
for f in fields(self):
|
|
50
|
+
value = getattr(self, f.name)
|
|
51
|
+
if f.name == "unknown_min_free":
|
|
52
|
+
_check_unknown_min_free(value)
|
|
53
|
+
continue
|
|
54
|
+
if f.name == "probation":
|
|
55
|
+
if value not in ("untrusted_only", "all"):
|
|
56
|
+
raise ValueError(f"probation must be 'untrusted_only' or 'all', got {value!r}")
|
|
57
|
+
continue
|
|
58
|
+
if f.name == "stable_requires_epoch":
|
|
59
|
+
if not isinstance(value, bool):
|
|
60
|
+
raise TypeError("stable_requires_epoch must be a bool")
|
|
61
|
+
continue
|
|
62
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
63
|
+
raise TypeError(f"{f.name} must be a number, got {type(value).__name__}")
|
|
64
|
+
if not math.isfinite(value) or value < 0:
|
|
65
|
+
raise ValueError(f"{f.name} must be finite and non-negative, got {value!r}")
|
|
66
|
+
for name in (
|
|
67
|
+
"stable_min_steps",
|
|
68
|
+
"max_oom_retries",
|
|
69
|
+
"max_lost_retries",
|
|
70
|
+
"min_trusted_observations",
|
|
71
|
+
"missing_samples_warn",
|
|
72
|
+
):
|
|
73
|
+
if not isinstance(getattr(self, name), int):
|
|
74
|
+
raise TypeError(f"{name} must be an int")
|
|
75
|
+
if self.safety_margin < 1:
|
|
76
|
+
raise ValueError("safety_margin must be >= 1")
|
|
77
|
+
if self.trusted_margin < 1:
|
|
78
|
+
raise ValueError("trusted_margin must be >= 1")
|
|
79
|
+
if self.oom_growth <= 1:
|
|
80
|
+
raise ValueError("oom_growth must be > 1, otherwise an OOM teaches nothing")
|
|
81
|
+
if self.min_warmup_s > self.max_warmup_s:
|
|
82
|
+
raise ValueError("min_warmup_s must be <= max_warmup_s")
|
|
83
|
+
if self.foreign_tolerance >= 1:
|
|
84
|
+
raise ValueError("foreign_tolerance must be < 1")
|
|
85
|
+
if self.vacate_headroom >= 1:
|
|
86
|
+
raise ValueError("vacate_headroom must be < 1")
|
|
87
|
+
|
|
88
|
+
def unknown_min_free_for(self, capacity: Resources) -> Resources:
|
|
89
|
+
"""``unknown_min_free`` on a device: absolute, or a fraction of its capacity."""
|
|
90
|
+
if isinstance(self.unknown_min_free, Resources):
|
|
91
|
+
return self.unknown_min_free
|
|
92
|
+
return capacity.scale(self.unknown_min_free)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _check_unknown_min_free(value: object) -> None:
|
|
96
|
+
if isinstance(value, Resources):
|
|
97
|
+
return
|
|
98
|
+
if isinstance(value, bool) or not isinstance(value, int | float):
|
|
99
|
+
raise TypeError("unknown_min_free must be Resources or a fraction of capacity")
|
|
100
|
+
if not 0 <= value <= 1:
|
|
101
|
+
raise ValueError(f"unknown_min_free as a fraction must be in [0, 1], got {value!r}")
|