stageload 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stageload/__init__.py ADDED
@@ -0,0 +1,23 @@
1
+ """Load a multi-stage PyTorch pipeline one stage at a time."""
2
+
3
+ from .events import Event, EventSink, ListSink
4
+ from .hooks import on_call
5
+ from .meter import MemoryMeter
6
+ from .registry import StagedModels
7
+ from .release import ReleasedModuleError, ReleaseStats, release
8
+ from .share import share
9
+
10
+ __version__ = "0.1.0"
11
+
12
+ __all__ = [
13
+ "Event",
14
+ "EventSink",
15
+ "ListSink",
16
+ "MemoryMeter",
17
+ "ReleaseStats",
18
+ "ReleasedModuleError",
19
+ "StagedModels",
20
+ "on_call",
21
+ "release",
22
+ "share",
23
+ ]
stageload/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
stageload/cli.py ADDED
@@ -0,0 +1,108 @@
1
+ """`stageload guard` and `stageload summary`."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import re
8
+ from collections.abc import Callable, Sequence
9
+ from typing import Any
10
+
11
+ from .guard import GuardConfig, run_guarded
12
+ from .trace import read_trace, summarize
13
+ from .units import parse_duration, parse_size
14
+
15
+
16
+ def _arg(parse: Callable[[str], Any]) -> Callable[[str], Any]:
17
+ def convert(text: str) -> Any:
18
+ try:
19
+ return parse(text)
20
+ except ValueError as error:
21
+ raise argparse.ArgumentTypeError(str(error)) from error
22
+
23
+ return convert
24
+
25
+
26
+ def _percent(text: str) -> int:
27
+ try:
28
+ value = int(text)
29
+ except ValueError as error:
30
+ raise argparse.ArgumentTypeError(f"not a whole percent: {text!r}") from error
31
+ if not 0 <= value <= 100:
32
+ raise argparse.ArgumentTypeError(f"percent out of range 0-100: {value}")
33
+ return value
34
+
35
+
36
+ def _positive_duration(text: str) -> float:
37
+ value = _arg(parse_duration)(text)
38
+ if value <= 0:
39
+ raise argparse.ArgumentTypeError(f"must be more than zero: {text!r}")
40
+ return value
41
+
42
+
43
+ def _regex(text: str) -> str:
44
+ try:
45
+ re.compile(text)
46
+ except re.error as error:
47
+ raise argparse.ArgumentTypeError(f"bad regex {text!r}: {error}") from error
48
+ return text
49
+
50
+
51
+ def _parser() -> argparse.ArgumentParser:
52
+ parser = argparse.ArgumentParser(prog="stageload")
53
+ sub = parser.add_subparsers(dest="command", required=True)
54
+ guard = sub.add_parser(
55
+ "guard", help="run one command when there is room and stop it before swap runs away"
56
+ )
57
+ guard.add_argument(
58
+ "--wait-free", type=_percent, default=40, metavar="PERCENT",
59
+ help="memory available before starting (default 40)",
60
+ )
61
+ guard.add_argument(
62
+ "--swap-budget", type=_arg(parse_size), default="8G",
63
+ help="swap the command may add over what was used at its start (default 8G)",
64
+ )
65
+ guard.add_argument(
66
+ "--swap-limit", type=_arg(parse_size), default=None,
67
+ help="absolute swap ceiling, also checked before starting",
68
+ )
69
+ guard.add_argument(
70
+ "--kill-free", type=_percent, default=10, metavar="PERCENT",
71
+ help="stop the command after two samples under this (default 10)",
72
+ )
73
+ guard.add_argument(
74
+ "--busy", action="append", type=_regex, default=[], metavar="REGEX",
75
+ help="do not start while a process matches (repeatable)",
76
+ )
77
+ guard.add_argument(
78
+ "--quiet-for", type=_arg(parse_duration), default="1m",
79
+ help="with --busy: start only after no busy process was seen for this long (default 1m)",
80
+ )
81
+ guard.add_argument("--start-timeout", type=_arg(parse_duration), default="30m")
82
+ guard.add_argument("--poll", type=_positive_duration, default="5s")
83
+ guard.add_argument("cmd", nargs=argparse.REMAINDER, help="-- command ...")
84
+ summary = sub.add_parser("summary", help="print the summary of a trace file as JSON")
85
+ summary.add_argument("trace")
86
+ return parser
87
+
88
+
89
+ def main(argv: Sequence[str] | None = None) -> int:
90
+ parser = _parser()
91
+ args = parser.parse_args(argv)
92
+ if args.command == "summary":
93
+ print(json.dumps(summarize(read_trace(args.trace)), indent=2))
94
+ return 0
95
+ cmd = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd
96
+ if not cmd:
97
+ parser.error("guard needs a command after --")
98
+ config = GuardConfig(
99
+ wait_free=args.wait_free,
100
+ swap_budget=args.swap_budget,
101
+ swap_limit=args.swap_limit,
102
+ kill_free=args.kill_free,
103
+ busy=tuple(args.busy),
104
+ quiet_for=args.quiet_for,
105
+ start_timeout=args.start_timeout,
106
+ poll=args.poll,
107
+ )
108
+ return run_guarded(cmd, config)
stageload/events.py ADDED
@@ -0,0 +1,41 @@
1
+ """Events that stageload emits while it loads, releases and changes stages."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from dataclasses import asdict, dataclass, field
7
+ from typing import Any, Protocol
8
+
9
+ EVENT_KINDS = ("load", "reload", "release", "stage", "mark")
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class Event:
14
+ kind: str
15
+ name: str | None = None
16
+ stage: str | None = None
17
+ nbytes: int | None = None
18
+ seconds: float | None = None
19
+ note: str | None = None
20
+ t: float = field(default_factory=time.monotonic)
21
+
22
+ def __post_init__(self) -> None:
23
+ if self.kind not in EVENT_KINDS:
24
+ raise ValueError(f"unknown event kind {self.kind!r}; expected one of {EVENT_KINDS}")
25
+
26
+ def as_dict(self) -> dict[str, Any]:
27
+ return {key: value for key, value in asdict(self).items() if value is not None}
28
+
29
+
30
+ class EventSink(Protocol):
31
+ def emit(self, event: Event) -> None: ...
32
+
33
+
34
+ class ListSink:
35
+ """Keeps events in a list, for tests and quick summaries."""
36
+
37
+ def __init__(self) -> None:
38
+ self.events: list[Event] = []
39
+
40
+ def emit(self, event: Event) -> None:
41
+ self.events.append(event)
stageload/guard.py ADDED
@@ -0,0 +1,328 @@
1
+ """Start one heavy command only when there is room, and stop it before it takes the machine down.
2
+
3
+ Unlike a system-wide OOM killer, the guard needs no root, and the only processes it ever stops
4
+ are the command it started and that command's process group. If the guard itself gets SIGINT,
5
+ SIGTERM or SIGHUP, it stops the command first, unless it was started to ignore that signal (as
6
+ ``nohup`` does for SIGHUP). SIGKILL cannot be caught: a command whose guard was killed that way
7
+ keeps running.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import math
13
+ import os
14
+ import re
15
+ import signal
16
+ import subprocess
17
+ import sys
18
+ import threading
19
+ import time
20
+ from collections.abc import Callable, Iterable, Sequence
21
+ from dataclasses import dataclass
22
+ from datetime import datetime, timezone
23
+ from types import FrameType
24
+ from typing import Any
25
+
26
+ EXIT_KILLED = 3
27
+ EXIT_NO_ROOM = 4
28
+ EXIT_CANNOT_START = 127
29
+ GB = 2**30
30
+
31
+ Process = tuple[int, int, str] # pid, parent pid, command line
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class GuardConfig:
36
+ wait_free: int = 40
37
+ swap_budget: int = 8 * GB
38
+ swap_limit: int | None = None
39
+ kill_free: int = 10
40
+ busy: tuple[str, ...] = ()
41
+ quiet_for: float = 0.0
42
+ start_timeout: float = 1800.0
43
+ poll: float = 5.0
44
+ grace: float = 5.0
45
+
46
+
47
+ def _stderr(line: str) -> None:
48
+ stamp = datetime.now(timezone.utc).strftime("%H:%M:%SZ")
49
+ print(f"{stamp} {line}", file=sys.stderr, flush=True)
50
+
51
+
52
+ @dataclass
53
+ class GuardIO:
54
+ level: Callable[[], int]
55
+ swap_used: Callable[[], int]
56
+ processes: Callable[[], list[Process]]
57
+ sleep: Callable[[float], None] = time.sleep
58
+ clock: Callable[[], float] = time.monotonic
59
+ log: Callable[[str], None] = _stderr
60
+
61
+
62
+ def list_processes() -> list[Process]:
63
+ out = subprocess.run(
64
+ ["ps", "-axo", "pid=,ppid=,command="], capture_output=True, text=True, check=True
65
+ ).stdout
66
+ rows = []
67
+ for line in out.splitlines():
68
+ parts = line.split(None, 2)
69
+ if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
70
+ rows.append((int(parts[0]), int(parts[1]), parts[2] if len(parts) == 3 else ""))
71
+ return rows
72
+
73
+
74
+ def system_io() -> GuardIO:
75
+ from .meter import memorystatus_level, swap_used
76
+
77
+ return GuardIO(level=memorystatus_level, swap_used=swap_used, processes=list_processes)
78
+
79
+
80
+ def ancestors(processes: Iterable[Process], pid: int) -> set[int]:
81
+ """``pid`` and every process above it. Their command lines contain the guard's own
82
+ arguments, so they always match its ``--busy`` patterns."""
83
+ parent = {p: pp for p, pp, _ in processes}
84
+ chain = {pid}
85
+ while pid in parent and parent[pid] not in chain and parent[pid] > 1:
86
+ pid = parent[pid]
87
+ chain.add(pid)
88
+ return chain
89
+
90
+
91
+ def busy_processes(
92
+ regexes: Sequence[re.Pattern[str]], processes: Iterable[Process], exclude: set[int]
93
+ ) -> list[str]:
94
+ """Command lines that match a pattern, like ``pgrep -f``, leaving out ``exclude``."""
95
+ return [
96
+ command
97
+ for pid, _, command in processes
98
+ if pid not in exclude and any(r.search(command) for r in regexes)
99
+ ]
100
+
101
+
102
+ def _gb(nbytes: int) -> str:
103
+ return f"{nbytes / GB:.1f} GB"
104
+
105
+
106
+ def _no_room_reasons(
107
+ config: GuardConfig,
108
+ io: GuardIO,
109
+ regexes: Sequence[re.Pattern[str]],
110
+ quiet: dict[str, float | None],
111
+ ) -> list[str]:
112
+ reasons = []
113
+ level = io.level()
114
+ swap = io.swap_used()
115
+ if level < config.wait_free:
116
+ reasons.append(f"memory available {level}% < {config.wait_free}%")
117
+ if config.swap_limit is not None and swap + config.swap_budget > config.swap_limit:
118
+ reasons.append(
119
+ f"swap {_gb(swap)} + budget {_gb(config.swap_budget)} > limit {_gb(config.swap_limit)}"
120
+ )
121
+ if regexes:
122
+ processes = io.processes()
123
+ busy = busy_processes(regexes, processes, ancestors(processes, os.getpid()))
124
+ if busy:
125
+ quiet["since"] = None
126
+ reasons.append("busy: " + "; ".join(command[:80] for command in busy[:3]))
127
+ elif config.quiet_for > 0:
128
+ # a chain of jobs leaves short gaps between them: on 2026-10-05 a run started in
129
+ # such a gap, and the next video job took swap past its budget within 20 s
130
+ if quiet["since"] is None:
131
+ quiet["since"] = io.clock()
132
+ quiet_so_far = io.clock() - quiet["since"]
133
+ if quiet_so_far < config.quiet_for:
134
+ reasons.append(f"quiet for {quiet_so_far:.0f} s of {config.quiet_for:.0f} s")
135
+ return reasons
136
+
137
+
138
+ class _Signals:
139
+ """While installed, SIGINT, SIGTERM and SIGHUP are recorded instead of acting at once, so the
140
+ guard can stop its command whatever it was doing when one arrived. A signal the guard was
141
+ started to ignore stays ignored. Handlers can only be installed from the main thread."""
142
+
143
+ HANDLED = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)
144
+
145
+ def __init__(self) -> None:
146
+ self.first: int | None = None
147
+ self.count = 0
148
+ self._previous: dict[int, Any] = {}
149
+
150
+ def __enter__(self) -> _Signals:
151
+ if threading.current_thread() is threading.main_thread():
152
+ for signum in self.HANDLED:
153
+ if signal.getsignal(signum) is not signal.SIG_IGN:
154
+ self._previous[signum] = signal.signal(signum, self._record)
155
+ return self
156
+
157
+ def __exit__(self, *exc: object) -> None:
158
+ for signum, handler in self._previous.items():
159
+ signal.signal(signum, handler)
160
+
161
+ def _record(self, signum: int, frame: FrameType | None) -> None:
162
+ if self.first is None:
163
+ self.first = signum
164
+ self.count += 1
165
+
166
+
167
+ _SLICE = 0.2 # seconds: how long the guard can take to notice a signal
168
+
169
+
170
+ def _nap(io: GuardIO, seconds: float, signals: _Signals | None) -> None:
171
+ slices = max(1, math.ceil(seconds / _SLICE))
172
+ for _ in range(slices):
173
+ if signals is not None and signals.count:
174
+ return
175
+ io.sleep(seconds / slices)
176
+
177
+
178
+ def wait_for_room(config: GuardConfig, io: GuardIO, signals: _Signals | None = None) -> bool:
179
+ """True once there is room; False after ``start_timeout``, or as soon as a signal arrives."""
180
+ regexes = [re.compile(pattern) for pattern in config.busy]
181
+ deadline = io.clock() + config.start_timeout
182
+ last = ""
183
+ quiet: dict[str, float | None] = {"since": None}
184
+ while True:
185
+ if signals is not None and signals.count:
186
+ return False
187
+ reasons = _no_room_reasons(config, io, regexes, quiet)
188
+ if not reasons:
189
+ return True
190
+ reason = ", ".join(reasons)
191
+ kind = reason.split(" for ")[0] if reason.startswith("quiet") else reason
192
+ if kind != last:
193
+ io.log(f"guard: waiting ({reason})")
194
+ last = kind
195
+ if io.clock() >= deadline:
196
+ io.log(f"guard: no room after {config.start_timeout:.0f} s ({reason})")
197
+ return False
198
+ _nap(io, config.poll, signals)
199
+
200
+
201
+ def _group_alive(pgid: int) -> bool:
202
+ try:
203
+ os.killpg(pgid, 0)
204
+ except ProcessLookupError:
205
+ return False
206
+ except PermissionError:
207
+ return True
208
+ return True
209
+
210
+
211
+ def _stop_group(
212
+ proc: subprocess.Popen[bytes], grace: float, signals: _Signals | None = None
213
+ ) -> None:
214
+ """SIGTERM to the command's group, then SIGKILL to whatever is left of it once ``grace``
215
+ seconds have passed, or at once if the guard gets another signal meanwhile.
216
+
217
+ The grace is for the whole group: a shell that exits at once does not cut short the cleanup
218
+ of the job it started.
219
+ """
220
+ pgid = proc.pid
221
+ seen = signals.count if signals is not None else 0
222
+ try:
223
+ os.killpg(pgid, signal.SIGTERM)
224
+ except ProcessLookupError:
225
+ proc.wait()
226
+ return
227
+ deadline = time.monotonic() + grace
228
+ while time.monotonic() < deadline:
229
+ proc.poll() # reap the leader, or it stays in the group as a zombie
230
+ if not _group_alive(pgid):
231
+ break
232
+ if signals is not None and signals.count > seen:
233
+ break
234
+ time.sleep(0.05)
235
+ try:
236
+ os.killpg(pgid, signal.SIGKILL)
237
+ except ProcessLookupError:
238
+ pass
239
+ proc.wait()
240
+
241
+
242
+ def _wait(proc: subprocess.Popen[bytes], seconds: float, signals: _Signals) -> int | None:
243
+ """The command's exit code if it ends within ``seconds``; None if it is still running or a
244
+ signal arrived."""
245
+ deadline = time.monotonic() + seconds
246
+ while True:
247
+ remaining = deadline - time.monotonic()
248
+ try:
249
+ return proc.wait(timeout=max(0.0, min(_SLICE, remaining)))
250
+ except subprocess.TimeoutExpired:
251
+ if signals.count or remaining <= _SLICE:
252
+ return None
253
+
254
+
255
+ def _exit_code(code: int) -> int:
256
+ """A command ended by a signal reports -N; a shell reports 128+N."""
257
+ return 128 - code if code < 0 else code
258
+
259
+
260
+ def _signal_name(signum: int) -> str:
261
+ try:
262
+ return signal.Signals(signum).name
263
+ except ValueError:
264
+ return f"signal {signum}"
265
+
266
+
267
+ def run_guarded(cmd: Sequence[str], config: GuardConfig, io: GuardIO | None = None) -> int:
268
+ io = io or system_io()
269
+ with _Signals() as signals:
270
+ room = wait_for_room(config, io, signals)
271
+ if signals.first is not None:
272
+ io.log(f"guard: got {_signal_name(signals.first)} before starting the command")
273
+ return 128 + signals.first
274
+ if not room:
275
+ return EXIT_NO_ROOM
276
+ swap_at_start = io.swap_used()
277
+ limit = swap_at_start + config.swap_budget
278
+ if config.swap_limit is not None:
279
+ limit = min(limit, config.swap_limit)
280
+ io.log(
281
+ f"guard: start, swap {_gb(swap_at_start)}, stop above {_gb(limit)} "
282
+ f"or under {config.kill_free}% available twice"
283
+ )
284
+ try:
285
+ proc = subprocess.Popen(list(cmd), start_new_session=True)
286
+ except OSError as error:
287
+ io.log(f"guard: cannot start {cmd[0]!r}: {error.strerror or error}")
288
+ return EXIT_CANNOT_START
289
+ try:
290
+ return _watch(proc, config, io, signals, limit)
291
+ except BaseException:
292
+ # the command cannot be watched any more, so it does not keep running unwatched
293
+ _stop_group(proc, config.grace, signals)
294
+ raise
295
+
296
+
297
+ def _watch(
298
+ proc: subprocess.Popen[bytes],
299
+ config: GuardConfig,
300
+ io: GuardIO,
301
+ signals: _Signals,
302
+ limit: int,
303
+ ) -> int:
304
+ low = 0
305
+ while True:
306
+ if signals.first is not None:
307
+ io.log(
308
+ f"guard: got {_signal_name(signals.first)}, stopping pid {proc.pid} and its group"
309
+ )
310
+ _stop_group(proc, config.grace, signals)
311
+ return 128 + signals.first
312
+ code = _wait(proc, config.poll, signals)
313
+ if code is not None:
314
+ io.log(f"guard: exit {_exit_code(code)}")
315
+ return _exit_code(code)
316
+ if signals.first is not None:
317
+ continue
318
+ level = io.level()
319
+ swap = io.swap_used()
320
+ low = low + 1 if level < config.kill_free else 0
321
+ if swap > limit or low >= 2:
322
+ if swap > limit:
323
+ why = f"swap {_gb(swap)} > {_gb(limit)}"
324
+ else:
325
+ why = f"memory available {level}% < {config.kill_free}% twice"
326
+ io.log(f"guard: killing pid {proc.pid} and its group ({why})")
327
+ _stop_group(proc, config.grace, signals)
328
+ return EXIT_KILLED
stageload/hooks.py ADDED
@@ -0,0 +1,50 @@
1
+ """Run callbacks around one method of one object."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import functools
6
+ from collections.abc import Callable
7
+ from typing import Any
8
+
9
+
10
+ def on_call(
11
+ obj: Any,
12
+ method: str,
13
+ *,
14
+ before: Callable[..., None] | None = None,
15
+ after: Callable[..., None] | None = None,
16
+ ) -> Callable[[], None]:
17
+ """Wrap ``obj.method`` on this instance only; return a function that removes the wrapper.
18
+
19
+ ``before(*args, **kwargs)`` runs before the call, ``after(result, *args, **kwargs)`` after
20
+ it. Wrappers stack; remove them in reverse order.
21
+ """
22
+ if before is None and after is None:
23
+ raise ValueError("on_call needs a before or an after callback")
24
+ original = getattr(obj, method)
25
+ had_own = method in vars(obj)
26
+ previous = vars(obj).get(method)
27
+
28
+ @functools.wraps(original)
29
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
30
+ if before is not None:
31
+ before(*args, **kwargs)
32
+ result = original(*args, **kwargs)
33
+ if after is not None:
34
+ after(result, *args, **kwargs)
35
+ return result
36
+
37
+ setattr(obj, method, wrapper)
38
+
39
+ def uninstall() -> None:
40
+ if vars(obj).get(method) is not wrapper:
41
+ raise RuntimeError(
42
+ f"another wrapper was installed on {method!r} after this one; "
43
+ "uninstall them in reverse order"
44
+ )
45
+ if had_own:
46
+ setattr(obj, method, previous)
47
+ else:
48
+ delattr(obj, method)
49
+
50
+ return uninstall