rampgate 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rampgate/__init__.py ADDED
@@ -0,0 +1,234 @@
1
+ """rampgate: observe-then-reserve admission scheduler."""
2
+
3
+ from rampgate.actions import (
4
+ Action,
5
+ CapacityExceeded,
6
+ Fail,
7
+ FailReason,
8
+ ForeignUsageHigh,
9
+ GroupCompleted,
10
+ GroupOutcome,
11
+ HintExceeded,
12
+ JobNotObserved,
13
+ MarkStable,
14
+ NoUsageObserved,
15
+ Overcommitted,
16
+ QueueStopped,
17
+ Requeue,
18
+ RequeueReason,
19
+ Start,
20
+ Stop,
21
+ UncleanStop,
22
+ Unreconciled,
23
+ UpdateReservation,
24
+ Warn,
25
+ WarnCleared,
26
+ WarningKind,
27
+ is_edge_triggered,
28
+ key_id,
29
+ warning_key,
30
+ )
31
+ from rampgate.config import Config
32
+ from rampgate.decide import (
33
+ PROBATION,
34
+ Admission,
35
+ Decision,
36
+ DeviceLoad,
37
+ ProbationAdmission,
38
+ decide,
39
+ device_loads,
40
+ measured_free,
41
+ )
42
+ from rampgate.engine import Engine, JobSample
43
+ from rampgate.estimators import (
44
+ EstimateResult,
45
+ Estimator,
46
+ HistoryEntry,
47
+ HistoryEstimator,
48
+ HistoryStore,
49
+ InMemoryHistoryStore,
50
+ NullEstimator,
51
+ StaticEstimator,
52
+ apply_history,
53
+ resolve_estimates,
54
+ )
55
+ from rampgate.events import (
56
+ UNSET,
57
+ CancelRequested,
58
+ DeviceAdded,
59
+ DeviceObserved,
60
+ DeviceRemoved,
61
+ Event,
62
+ JobExited,
63
+ JobObserved,
64
+ JobStarted,
65
+ JobSubmitted,
66
+ JobUpdated,
67
+ Paused,
68
+ PreemptRequested,
69
+ ResumeRequested,
70
+ Unpaused,
71
+ )
72
+ from rampgate.explain import (
73
+ AwaitingResume,
74
+ BlockedByForeignUsage,
75
+ DeviceHeldFor,
76
+ EnginePaused,
77
+ Explanation,
78
+ GangPartiallyBlocked,
79
+ GroupBarrier,
80
+ InsufficientResources,
81
+ RejectedByAdmission,
82
+ UnknownEstimateBlocked,
83
+ WaitingForIsolation,
84
+ WaitingForWarmup,
85
+ )
86
+ from rampgate.model import (
87
+ ACTIVE_STATES,
88
+ TERMINAL_STATES,
89
+ Device,
90
+ DeviceId,
91
+ DeviceState,
92
+ ExitKind,
93
+ GroupId,
94
+ HistoryRecord,
95
+ Job,
96
+ JobId,
97
+ JobRuntime,
98
+ JobState,
99
+ Snapshot,
100
+ State,
101
+ StopReason,
102
+ )
103
+ from rampgate.policies import (
104
+ BasePolicy,
105
+ FailurePolicy,
106
+ FifoPolicy,
107
+ OrderingPolicy,
108
+ PriorityPolicy,
109
+ StagePolicy,
110
+ StateView,
111
+ )
112
+ from rampgate.reducer import IgnoredEvent, IgnoreReason, InvalidTransition, Message, drain, reduce
113
+ from rampgate.resources import GiB, KiB, MiB, Number, Resources, TiB
114
+ from rampgate.serialization import UnsupportedSchema, dump_state, load_state
115
+ from rampgate.stability import is_peak_growth, is_stable
116
+
117
+ __version__ = "0.1.0"
118
+
119
+ __all__ = [
120
+ "ACTIVE_STATES",
121
+ "PROBATION",
122
+ "TERMINAL_STATES",
123
+ "UNSET",
124
+ "Action",
125
+ "Admission",
126
+ "AwaitingResume",
127
+ "BasePolicy",
128
+ "BlockedByForeignUsage",
129
+ "CancelRequested",
130
+ "CapacityExceeded",
131
+ "Config",
132
+ "Decision",
133
+ "Device",
134
+ "DeviceAdded",
135
+ "DeviceHeldFor",
136
+ "DeviceId",
137
+ "DeviceLoad",
138
+ "DeviceObserved",
139
+ "DeviceRemoved",
140
+ "DeviceState",
141
+ "Engine",
142
+ "EnginePaused",
143
+ "EstimateResult",
144
+ "Estimator",
145
+ "Event",
146
+ "ExitKind",
147
+ "Explanation",
148
+ "Fail",
149
+ "FailReason",
150
+ "FailurePolicy",
151
+ "FifoPolicy",
152
+ "ForeignUsageHigh",
153
+ "GangPartiallyBlocked",
154
+ "GiB",
155
+ "GroupBarrier",
156
+ "GroupCompleted",
157
+ "GroupId",
158
+ "GroupOutcome",
159
+ "HintExceeded",
160
+ "HistoryEntry",
161
+ "HistoryEstimator",
162
+ "HistoryRecord",
163
+ "HistoryStore",
164
+ "IgnoreReason",
165
+ "IgnoredEvent",
166
+ "InMemoryHistoryStore",
167
+ "InsufficientResources",
168
+ "InvalidTransition",
169
+ "Job",
170
+ "JobExited",
171
+ "JobId",
172
+ "JobNotObserved",
173
+ "JobObserved",
174
+ "JobRuntime",
175
+ "JobSample",
176
+ "JobStarted",
177
+ "JobState",
178
+ "JobSubmitted",
179
+ "JobUpdated",
180
+ "KiB",
181
+ "MarkStable",
182
+ "Message",
183
+ "MiB",
184
+ "NoUsageObserved",
185
+ "NullEstimator",
186
+ "Number",
187
+ "OrderingPolicy",
188
+ "Overcommitted",
189
+ "Paused",
190
+ "PreemptRequested",
191
+ "PriorityPolicy",
192
+ "ProbationAdmission",
193
+ "QueueStopped",
194
+ "RejectedByAdmission",
195
+ "Requeue",
196
+ "RequeueReason",
197
+ "Resources",
198
+ "ResumeRequested",
199
+ "Snapshot",
200
+ "StagePolicy",
201
+ "Start",
202
+ "State",
203
+ "StateView",
204
+ "StaticEstimator",
205
+ "Stop",
206
+ "StopReason",
207
+ "TiB",
208
+ "UncleanStop",
209
+ "UnknownEstimateBlocked",
210
+ "Unpaused",
211
+ "Unreconciled",
212
+ "UnsupportedSchema",
213
+ "UpdateReservation",
214
+ "WaitingForIsolation",
215
+ "WaitingForWarmup",
216
+ "Warn",
217
+ "WarnCleared",
218
+ "WarningKind",
219
+ "__version__",
220
+ "apply_history",
221
+ "decide",
222
+ "device_loads",
223
+ "drain",
224
+ "dump_state",
225
+ "is_edge_triggered",
226
+ "is_peak_growth",
227
+ "is_stable",
228
+ "key_id",
229
+ "load_state",
230
+ "measured_free",
231
+ "reduce",
232
+ "resolve_estimates",
233
+ "warning_key",
234
+ ]
rampgate/actions.py ADDED
@@ -0,0 +1,316 @@
1
+ """Output actions (docs/concepts.md), each with a deterministic ``action_id``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import dataclasses
6
+ import hashlib
7
+ import json
8
+ from collections.abc import Mapping
9
+ from dataclasses import dataclass, field
10
+ from enum import Enum, StrEnum
11
+ from typing import Any, TypeAlias
12
+
13
+ from rampgate.model import DeviceId, GroupId, JobId, StopReason, frozen_map
14
+ from rampgate.resources import Resources
15
+
16
+ __all__ = [
17
+ "Action",
18
+ "CapacityExceeded",
19
+ "Fail",
20
+ "FailReason",
21
+ "ForeignUsageHigh",
22
+ "GroupCompleted",
23
+ "GroupOutcome",
24
+ "HintExceeded",
25
+ "JobNotObserved",
26
+ "MarkStable",
27
+ "NoUsageObserved",
28
+ "Overcommitted",
29
+ "QueueStopped",
30
+ "Requeue",
31
+ "RequeueReason",
32
+ "Start",
33
+ "Stop",
34
+ "UncleanStop",
35
+ "Unreconciled",
36
+ "UpdateReservation",
37
+ "Warn",
38
+ "WarnCleared",
39
+ "WarningKind",
40
+ "canonical",
41
+ "is_edge_triggered",
42
+ "key_id",
43
+ "warning_key",
44
+ ]
45
+
46
+
47
+ class RequeueReason(StrEnum):
48
+ OOM = "OOM"
49
+ START_TIMEOUT = "START_TIMEOUT"
50
+ LOST = "LOST"
51
+ VACATED = "VACATED" # Rule V: stopped to relieve an overloaded device, back to PENDING
52
+
53
+
54
+ class FailReason(StrEnum):
55
+ OOM_RETRIES_EXHAUSTED = "OOM_RETRIES_EXHAUSTED"
56
+ LOST_RETRIES_EXHAUSTED = "LOST_RETRIES_EXHAUSTED"
57
+ EXCEEDS_DEVICE_CAPACITY = "EXCEEDS_DEVICE_CAPACITY"
58
+ EXITED_FAILURE = "EXITED_FAILURE"
59
+
60
+
61
+ class GroupOutcome(StrEnum):
62
+ OK = "OK"
63
+ HAD_FAILURES = "HAD_FAILURES"
64
+
65
+
66
+ def canonical(value: Any) -> Any:
67
+ """Convert a value made of dataclasses, enums, mappings and Resources into plain JSON data."""
68
+ if isinstance(value, Resources):
69
+ return {"__resources__": value.to_json()}
70
+ if isinstance(value, Enum):
71
+ return value.value
72
+ if dataclasses.is_dataclass(value) and not isinstance(value, type):
73
+ return {
74
+ "__type__": type(value).__name__,
75
+ **{f.name: canonical(getattr(value, f.name)) for f in dataclasses.fields(value)},
76
+ }
77
+ if isinstance(value, Mapping):
78
+ return {str(k): canonical(v) for k, v in sorted(value.items())}
79
+ if isinstance(value, tuple | list):
80
+ return [canonical(v) for v in value]
81
+ if isinstance(value, frozenset):
82
+ return sorted(canonical(v) for v in value)
83
+ if value is None or isinstance(value, bool | int | float | str):
84
+ return value
85
+ raise TypeError(f"cannot canonicalize {type(value).__name__}")
86
+
87
+
88
+ class _ActionMixin:
89
+ __slots__ = ()
90
+
91
+ @property
92
+ def action_id(self) -> str:
93
+ """Hash of the action's type and content (which includes the attempt, where relevant)."""
94
+ blob = json.dumps(canonical(self), sort_keys=True, separators=(",", ":"))
95
+ return hashlib.sha256(blob.encode()).hexdigest()[:32]
96
+
97
+
98
+ @dataclass(frozen=True, slots=True)
99
+ class Start(_ActionMixin):
100
+ job_id: JobId
101
+ attempt: int
102
+ placement: tuple[DeviceId, ...]
103
+ resume: bool
104
+ admitted_free: Mapping[DeviceId, Resources]
105
+ issued_at: float
106
+ estimate: Mapping[DeviceId, Resources | None] = field(default_factory=frozen_map)
107
+ trusted: frozenset[DeviceId] = frozenset()
108
+ # Per trusted device: complete observations behind the estimate (rule A, docs/algorithm.md).
109
+ observations: Mapping[DeviceId, int] = field(default_factory=frozen_map)
110
+
111
+
112
+ @dataclass(frozen=True, slots=True)
113
+ class Stop(_ActionMixin):
114
+ job_id: JobId
115
+ attempt: int
116
+ reason: StopReason
117
+ grace_s: float
118
+ # Rule V (docs/algorithm.md): stopped by rampgate itself; the job returns to PENDING after exit.
119
+ vacate: bool = False
120
+
121
+
122
+ @dataclass(frozen=True, slots=True)
123
+ class MarkStable(_ActionMixin):
124
+ job_id: JobId
125
+ attempt: int
126
+ reservation: Mapping[DeviceId, Resources]
127
+ issued_at: float | None = None # decide's ``now``; the probation slot is kept until then
128
+
129
+
130
+ @dataclass(frozen=True, slots=True)
131
+ class UpdateReservation(_ActionMixin):
132
+ job_id: JobId
133
+ attempt: int
134
+ reservation: Mapping[DeviceId, Resources]
135
+
136
+
137
+ @dataclass(frozen=True, slots=True)
138
+ class Requeue(_ActionMixin):
139
+ job_id: JobId
140
+ next_attempt: int
141
+ new_estimate: Mapping[DeviceId, Resources | None]
142
+ reason: RequeueReason
143
+
144
+
145
+ @dataclass(frozen=True, slots=True)
146
+ class Fail(_ActionMixin):
147
+ job_id: JobId
148
+ attempt: int
149
+ reason: FailReason
150
+
151
+
152
+ @dataclass(frozen=True, slots=True)
153
+ class GroupCompleted(_ActionMixin):
154
+ group: GroupId
155
+ outcome: GroupOutcome
156
+
157
+
158
+ @dataclass(frozen=True, slots=True)
159
+ class CapacityExceeded:
160
+ """A candidate's estimate exceeds the device's capacity: it can never fit there."""
161
+
162
+ group: GroupId | None
163
+ device_id: DeviceId
164
+ estimated: Resources
165
+ available: Resources
166
+ job_id: JobId
167
+
168
+
169
+ @dataclass(frozen=True, slots=True)
170
+ class HintExceeded:
171
+ """A running job's observed peak exceeds its declared hint (informational, per attempt)."""
172
+
173
+ job_id: JobId
174
+ attempt: int
175
+ device_id: DeviceId
176
+ hint: Resources
177
+ observed: Resources
178
+
179
+
180
+ @dataclass(frozen=True, slots=True)
181
+ class NoUsageObserved:
182
+ """A WARMING job that reports no steps reached ``max_warmup_s`` without its usage on
183
+ the device ever exceeding the device's ``stable_min_usage``: Rule S keeps it WARMING."""
184
+
185
+ job_id: JobId
186
+ attempt: int
187
+ device_id: DeviceId
188
+
189
+
190
+ @dataclass(frozen=True, slots=True)
191
+ class JobNotObserved:
192
+ """A running job has not been observed on a device for ``missing_samples_warn``
193
+ consecutive device samples (its last known usage is still used)."""
194
+
195
+ job_id: JobId
196
+ attempt: int
197
+ device_id: DeviceId
198
+
199
+
200
+ @dataclass(frozen=True, slots=True)
201
+ class Overcommitted:
202
+ """``sum(R) > C - F`` on a device; ``capacity`` is ``C - F``."""
203
+
204
+ device_id: DeviceId
205
+ reserved: Resources
206
+ capacity: Resources
207
+
208
+
209
+ @dataclass(frozen=True, slots=True)
210
+ class ForeignUsageHigh:
211
+ """Foreign usage keeps candidates from fitting on a device; they are listed when raised."""
212
+
213
+ device_id: DeviceId
214
+ foreign: Resources
215
+ blocked_job_ids: tuple[JobId, ...]
216
+
217
+
218
+ @dataclass(frozen=True, slots=True)
219
+ class UncleanStop:
220
+ """A stopped job exited non-zero: work since its last checkpoint may be lost."""
221
+
222
+ job_id: JobId
223
+ attempt: int
224
+
225
+
226
+ @dataclass(frozen=True, slots=True)
227
+ class Unreconciled:
228
+ """After a restart, nobody reported on this running job within ``reconcile_grace_s``."""
229
+
230
+ job_id: JobId
231
+ attempt: int
232
+
233
+
234
+ @dataclass(frozen=True, slots=True)
235
+ class QueueStopped:
236
+ """A job of a STOP_QUEUE group failed; reducing this warning pauses the engine."""
237
+
238
+ group: GroupId
239
+ job_id: JobId
240
+
241
+
242
+ WarningKind: TypeAlias = (
243
+ CapacityExceeded
244
+ | Overcommitted
245
+ | ForeignUsageHigh
246
+ | UncleanStop
247
+ | QueueStopped
248
+ | Unreconciled
249
+ | HintExceeded
250
+ | NoUsageObserved
251
+ | JobNotObserved
252
+ )
253
+
254
+
255
+ EDGE_TRIGGERED: tuple[type, ...] = (
256
+ CapacityExceeded,
257
+ Overcommitted,
258
+ ForeignUsageHigh,
259
+ Unreconciled,
260
+ HintExceeded,
261
+ NoUsageObserved,
262
+ JobNotObserved,
263
+ )
264
+
265
+
266
+ def warning_key(w: WarningKind) -> tuple[str, tuple[str, ...]]:
267
+ """``(kind, key)`` of a warning (docs/concepts.md warning table)."""
268
+ kind = type(w).__name__
269
+ match w:
270
+ case CapacityExceeded():
271
+ return kind, (w.job_id, w.device_id)
272
+ case Overcommitted() | ForeignUsageHigh():
273
+ return kind, (w.device_id,)
274
+ case Unreconciled():
275
+ return kind, (w.job_id,)
276
+ case UncleanStop():
277
+ return kind, (w.job_id, str(w.attempt))
278
+ case HintExceeded() | NoUsageObserved() | JobNotObserved():
279
+ return kind, (w.job_id, str(w.attempt), w.device_id)
280
+ case QueueStopped():
281
+ return kind, (w.group, w.job_id)
282
+
283
+
284
+ def key_id(kind: str, key: tuple[str, ...]) -> str:
285
+ """A single string for a ``(kind, key)`` pair, used as a mapping key in the state."""
286
+ return json.dumps([kind, *key], separators=(",", ":"))
287
+
288
+
289
+ def is_edge_triggered(w: WarningKind) -> bool:
290
+ return isinstance(w, EDGE_TRIGGERED)
291
+
292
+
293
+ @dataclass(frozen=True, slots=True)
294
+ class Warn(_ActionMixin):
295
+ warning: WarningKind
296
+
297
+
298
+ @dataclass(frozen=True, slots=True)
299
+ class WarnCleared(_ActionMixin):
300
+ """The condition of an edge-triggered warning no longer holds."""
301
+
302
+ kind: str
303
+ key: tuple[str, ...]
304
+
305
+
306
+ Action: TypeAlias = (
307
+ Start
308
+ | Stop
309
+ | MarkStable
310
+ | UpdateReservation
311
+ | Requeue
312
+ | Fail
313
+ | GroupCompleted
314
+ | Warn
315
+ | WarnCleared
316
+ )
rampgate/config.py ADDED
@@ -0,0 +1,101 @@
1
+ """`Config`: every tunable threshold of the algorithm (docs/algorithm.md)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from dataclasses import dataclass, fields
7
+ from typing import Literal
8
+
9
+ from rampgate.resources import Resources
10
+
11
+ __all__ = ["Config"]
12
+
13
+
14
+ @dataclass(frozen=True, slots=True, kw_only=True)
15
+ class Config:
16
+ """Algorithm thresholds. Times are in seconds of the caller's clock."""
17
+
18
+ # Reservations based on one observation (a STABLE job's peak).
19
+ safety_margin: float = 1.15
20
+ # A trusted STARTING/WARMING job: its estimate is already a max of observations.
21
+ trusted_margin: float = 1.05
22
+ # ...but only with this many complete observations of its level-0 key (rule A).
23
+ min_trusted_observations: int = 3
24
+ min_warmup_s: float = 30
25
+ stable_window_s: float = 60
26
+ stable_min_steps: int = 20
27
+ stable_requires_epoch: bool = True
28
+ growth_tol: float = 0.02
29
+ max_warmup_s: float = 600
30
+ oom_growth: float = 1.25
31
+ max_oom_retries: int = 2
32
+ max_lost_retries: int = 3
33
+ preempt_grace_s: float = 120
34
+ foreign_window_s: float = 10
35
+ foreign_tolerance: float = 0.02
36
+ start_timeout_s: float = 300
37
+ reconcile_grace_s: float = 60
38
+ unknown_min_free: Resources | float = 0.10
39
+ # Rule P for jobs without a trusted estimate only (default), or for every job.
40
+ probation: Literal["untrusted_only", "all"] = "untrusted_only"
41
+ # Rule V: vacate when managed load exceeds C * (1 - headroom). From the simulator's
42
+ # noise bound (2 %); calibrate on real hardware.
43
+ vacate_headroom: float = 0.02
44
+ # Warn(JobNotObserved) after this many consecutive device samples without an
45
+ # observation of a running job there (0: never).
46
+ missing_samples_warn: int = 5
47
+
48
+ def __post_init__(self) -> None:
49
+ for f in fields(self):
50
+ value = getattr(self, f.name)
51
+ if f.name == "unknown_min_free":
52
+ _check_unknown_min_free(value)
53
+ continue
54
+ if f.name == "probation":
55
+ if value not in ("untrusted_only", "all"):
56
+ raise ValueError(f"probation must be 'untrusted_only' or 'all', got {value!r}")
57
+ continue
58
+ if f.name == "stable_requires_epoch":
59
+ if not isinstance(value, bool):
60
+ raise TypeError("stable_requires_epoch must be a bool")
61
+ continue
62
+ if isinstance(value, bool) or not isinstance(value, int | float):
63
+ raise TypeError(f"{f.name} must be a number, got {type(value).__name__}")
64
+ if not math.isfinite(value) or value < 0:
65
+ raise ValueError(f"{f.name} must be finite and non-negative, got {value!r}")
66
+ for name in (
67
+ "stable_min_steps",
68
+ "max_oom_retries",
69
+ "max_lost_retries",
70
+ "min_trusted_observations",
71
+ "missing_samples_warn",
72
+ ):
73
+ if not isinstance(getattr(self, name), int):
74
+ raise TypeError(f"{name} must be an int")
75
+ if self.safety_margin < 1:
76
+ raise ValueError("safety_margin must be >= 1")
77
+ if self.trusted_margin < 1:
78
+ raise ValueError("trusted_margin must be >= 1")
79
+ if self.oom_growth <= 1:
80
+ raise ValueError("oom_growth must be > 1, otherwise an OOM teaches nothing")
81
+ if self.min_warmup_s > self.max_warmup_s:
82
+ raise ValueError("min_warmup_s must be <= max_warmup_s")
83
+ if self.foreign_tolerance >= 1:
84
+ raise ValueError("foreign_tolerance must be < 1")
85
+ if self.vacate_headroom >= 1:
86
+ raise ValueError("vacate_headroom must be < 1")
87
+
88
+ def unknown_min_free_for(self, capacity: Resources) -> Resources:
89
+ """``unknown_min_free`` on a device: absolute, or a fraction of its capacity."""
90
+ if isinstance(self.unknown_min_free, Resources):
91
+ return self.unknown_min_free
92
+ return capacity.scale(self.unknown_min_free)
93
+
94
+
95
+ def _check_unknown_min_free(value: object) -> None:
96
+ if isinstance(value, Resources):
97
+ return
98
+ if isinstance(value, bool) or not isinstance(value, int | float):
99
+ raise TypeError("unknown_min_free must be Resources or a fraction of capacity")
100
+ if not 0 <= value <= 1:
101
+ raise ValueError(f"unknown_min_free as a fraction must be in [0, 1], got {value!r}")