stageload 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stageload/__init__.py +23 -0
- stageload/__main__.py +3 -0
- stageload/cli.py +108 -0
- stageload/events.py +41 -0
- stageload/guard.py +328 -0
- stageload/hooks.py +50 -0
- stageload/meter.py +225 -0
- stageload/pixal3d/__init__.py +1 -0
- stageload/pixal3d/__main__.py +3 -0
- stageload/pixal3d/cli.py +118 -0
- stageload/pixal3d/plan.py +64 -0
- stageload/pixal3d/port.py +141 -0
- stageload/pixal3d/probe.py +101 -0
- stageload/pixal3d/staged.py +279 -0
- stageload/registry.py +193 -0
- stageload/release.py +129 -0
- stageload/rng.py +45 -0
- stageload/share.py +55 -0
- stageload/trace.py +101 -0
- stageload/units.py +27 -0
- stageload-0.1.0.dist-info/METADATA +247 -0
- stageload-0.1.0.dist-info/RECORD +25 -0
- stageload-0.1.0.dist-info/WHEEL +4 -0
- stageload-0.1.0.dist-info/entry_points.txt +3 -0
- stageload-0.1.0.dist-info/licenses/LICENSE +21 -0
stageload/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Load a multi-stage PyTorch pipeline one stage at a time."""
|
|
2
|
+
|
|
3
|
+
from .events import Event, EventSink, ListSink
|
|
4
|
+
from .hooks import on_call
|
|
5
|
+
from .meter import MemoryMeter
|
|
6
|
+
from .registry import StagedModels
|
|
7
|
+
from .release import ReleasedModuleError, ReleaseStats, release
|
|
8
|
+
from .share import share
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Event",
|
|
14
|
+
"EventSink",
|
|
15
|
+
"ListSink",
|
|
16
|
+
"MemoryMeter",
|
|
17
|
+
"ReleaseStats",
|
|
18
|
+
"ReleasedModuleError",
|
|
19
|
+
"StagedModels",
|
|
20
|
+
"on_call",
|
|
21
|
+
"release",
|
|
22
|
+
"share",
|
|
23
|
+
]
|
stageload/__main__.py
ADDED
stageload/cli.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""`stageload guard` and `stageload summary`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .guard import GuardConfig, run_guarded
|
|
12
|
+
from .trace import read_trace, summarize
|
|
13
|
+
from .units import parse_duration, parse_size
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _arg(parse: Callable[[str], Any]) -> Callable[[str], Any]:
|
|
17
|
+
def convert(text: str) -> Any:
|
|
18
|
+
try:
|
|
19
|
+
return parse(text)
|
|
20
|
+
except ValueError as error:
|
|
21
|
+
raise argparse.ArgumentTypeError(str(error)) from error
|
|
22
|
+
|
|
23
|
+
return convert
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _percent(text: str) -> int:
|
|
27
|
+
try:
|
|
28
|
+
value = int(text)
|
|
29
|
+
except ValueError as error:
|
|
30
|
+
raise argparse.ArgumentTypeError(f"not a whole percent: {text!r}") from error
|
|
31
|
+
if not 0 <= value <= 100:
|
|
32
|
+
raise argparse.ArgumentTypeError(f"percent out of range 0-100: {value}")
|
|
33
|
+
return value
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _positive_duration(text: str) -> float:
|
|
37
|
+
value = _arg(parse_duration)(text)
|
|
38
|
+
if value <= 0:
|
|
39
|
+
raise argparse.ArgumentTypeError(f"must be more than zero: {text!r}")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _regex(text: str) -> str:
|
|
44
|
+
try:
|
|
45
|
+
re.compile(text)
|
|
46
|
+
except re.error as error:
|
|
47
|
+
raise argparse.ArgumentTypeError(f"bad regex {text!r}: {error}") from error
|
|
48
|
+
return text
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _parser() -> argparse.ArgumentParser:
|
|
52
|
+
parser = argparse.ArgumentParser(prog="stageload")
|
|
53
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
54
|
+
guard = sub.add_parser(
|
|
55
|
+
"guard", help="run one command when there is room and stop it before swap runs away"
|
|
56
|
+
)
|
|
57
|
+
guard.add_argument(
|
|
58
|
+
"--wait-free", type=_percent, default=40, metavar="PERCENT",
|
|
59
|
+
help="memory available before starting (default 40)",
|
|
60
|
+
)
|
|
61
|
+
guard.add_argument(
|
|
62
|
+
"--swap-budget", type=_arg(parse_size), default="8G",
|
|
63
|
+
help="swap the command may add over what was used at its start (default 8G)",
|
|
64
|
+
)
|
|
65
|
+
guard.add_argument(
|
|
66
|
+
"--swap-limit", type=_arg(parse_size), default=None,
|
|
67
|
+
help="absolute swap ceiling, also checked before starting",
|
|
68
|
+
)
|
|
69
|
+
guard.add_argument(
|
|
70
|
+
"--kill-free", type=_percent, default=10, metavar="PERCENT",
|
|
71
|
+
help="stop the command after two samples under this (default 10)",
|
|
72
|
+
)
|
|
73
|
+
guard.add_argument(
|
|
74
|
+
"--busy", action="append", type=_regex, default=[], metavar="REGEX",
|
|
75
|
+
help="do not start while a process matches (repeatable)",
|
|
76
|
+
)
|
|
77
|
+
guard.add_argument(
|
|
78
|
+
"--quiet-for", type=_arg(parse_duration), default="1m",
|
|
79
|
+
help="with --busy: start only after no busy process was seen for this long (default 1m)",
|
|
80
|
+
)
|
|
81
|
+
guard.add_argument("--start-timeout", type=_arg(parse_duration), default="30m")
|
|
82
|
+
guard.add_argument("--poll", type=_positive_duration, default="5s")
|
|
83
|
+
guard.add_argument("cmd", nargs=argparse.REMAINDER, help="-- command ...")
|
|
84
|
+
summary = sub.add_parser("summary", help="print the summary of a trace file as JSON")
|
|
85
|
+
summary.add_argument("trace")
|
|
86
|
+
return parser
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
90
|
+
parser = _parser()
|
|
91
|
+
args = parser.parse_args(argv)
|
|
92
|
+
if args.command == "summary":
|
|
93
|
+
print(json.dumps(summarize(read_trace(args.trace)), indent=2))
|
|
94
|
+
return 0
|
|
95
|
+
cmd = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd
|
|
96
|
+
if not cmd:
|
|
97
|
+
parser.error("guard needs a command after --")
|
|
98
|
+
config = GuardConfig(
|
|
99
|
+
wait_free=args.wait_free,
|
|
100
|
+
swap_budget=args.swap_budget,
|
|
101
|
+
swap_limit=args.swap_limit,
|
|
102
|
+
kill_free=args.kill_free,
|
|
103
|
+
busy=tuple(args.busy),
|
|
104
|
+
quiet_for=args.quiet_for,
|
|
105
|
+
start_timeout=args.start_timeout,
|
|
106
|
+
poll=args.poll,
|
|
107
|
+
)
|
|
108
|
+
return run_guarded(cmd, config)
|
stageload/events.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Events that stageload emits while it loads, releases and changes stages."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from typing import Any, Protocol
|
|
8
|
+
|
|
9
|
+
EVENT_KINDS = ("load", "reload", "release", "stage", "mark")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Event:
|
|
14
|
+
kind: str
|
|
15
|
+
name: str | None = None
|
|
16
|
+
stage: str | None = None
|
|
17
|
+
nbytes: int | None = None
|
|
18
|
+
seconds: float | None = None
|
|
19
|
+
note: str | None = None
|
|
20
|
+
t: float = field(default_factory=time.monotonic)
|
|
21
|
+
|
|
22
|
+
def __post_init__(self) -> None:
|
|
23
|
+
if self.kind not in EVENT_KINDS:
|
|
24
|
+
raise ValueError(f"unknown event kind {self.kind!r}; expected one of {EVENT_KINDS}")
|
|
25
|
+
|
|
26
|
+
def as_dict(self) -> dict[str, Any]:
|
|
27
|
+
return {key: value for key, value in asdict(self).items() if value is not None}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class EventSink(Protocol):
|
|
31
|
+
def emit(self, event: Event) -> None: ...
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ListSink:
|
|
35
|
+
"""Keeps events in a list, for tests and quick summaries."""
|
|
36
|
+
|
|
37
|
+
def __init__(self) -> None:
|
|
38
|
+
self.events: list[Event] = []
|
|
39
|
+
|
|
40
|
+
def emit(self, event: Event) -> None:
|
|
41
|
+
self.events.append(event)
|
stageload/guard.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Start one heavy command only when there is room, and stop it before it takes the machine down.
|
|
2
|
+
|
|
3
|
+
Unlike a system-wide OOM killer, the guard needs no root, and the only processes it ever stops
|
|
4
|
+
are the command it started and that command's process group. If the guard itself gets SIGINT,
|
|
5
|
+
SIGTERM or SIGHUP, it stops the command first, unless it was started to ignore that signal (as
|
|
6
|
+
``nohup`` does for SIGHUP). SIGKILL cannot be caught: a command whose guard was killed that way
|
|
7
|
+
keeps running.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import signal
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
from types import FrameType
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
EXIT_KILLED = 3
|
|
27
|
+
EXIT_NO_ROOM = 4
|
|
28
|
+
EXIT_CANNOT_START = 127
|
|
29
|
+
GB = 2**30
|
|
30
|
+
|
|
31
|
+
Process = tuple[int, int, str] # pid, parent pid, command line
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class GuardConfig:
|
|
36
|
+
wait_free: int = 40
|
|
37
|
+
swap_budget: int = 8 * GB
|
|
38
|
+
swap_limit: int | None = None
|
|
39
|
+
kill_free: int = 10
|
|
40
|
+
busy: tuple[str, ...] = ()
|
|
41
|
+
quiet_for: float = 0.0
|
|
42
|
+
start_timeout: float = 1800.0
|
|
43
|
+
poll: float = 5.0
|
|
44
|
+
grace: float = 5.0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _stderr(line: str) -> None:
|
|
48
|
+
stamp = datetime.now(timezone.utc).strftime("%H:%M:%SZ")
|
|
49
|
+
print(f"{stamp} {line}", file=sys.stderr, flush=True)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class GuardIO:
|
|
54
|
+
level: Callable[[], int]
|
|
55
|
+
swap_used: Callable[[], int]
|
|
56
|
+
processes: Callable[[], list[Process]]
|
|
57
|
+
sleep: Callable[[float], None] = time.sleep
|
|
58
|
+
clock: Callable[[], float] = time.monotonic
|
|
59
|
+
log: Callable[[str], None] = _stderr
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def list_processes() -> list[Process]:
|
|
63
|
+
out = subprocess.run(
|
|
64
|
+
["ps", "-axo", "pid=,ppid=,command="], capture_output=True, text=True, check=True
|
|
65
|
+
).stdout
|
|
66
|
+
rows = []
|
|
67
|
+
for line in out.splitlines():
|
|
68
|
+
parts = line.split(None, 2)
|
|
69
|
+
if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
|
|
70
|
+
rows.append((int(parts[0]), int(parts[1]), parts[2] if len(parts) == 3 else ""))
|
|
71
|
+
return rows
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def system_io() -> GuardIO:
|
|
75
|
+
from .meter import memorystatus_level, swap_used
|
|
76
|
+
|
|
77
|
+
return GuardIO(level=memorystatus_level, swap_used=swap_used, processes=list_processes)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def ancestors(processes: Iterable[Process], pid: int) -> set[int]:
|
|
81
|
+
"""``pid`` and every process above it. Their command lines contain the guard's own
|
|
82
|
+
arguments, so they always match its ``--busy`` patterns."""
|
|
83
|
+
parent = {p: pp for p, pp, _ in processes}
|
|
84
|
+
chain = {pid}
|
|
85
|
+
while pid in parent and parent[pid] not in chain and parent[pid] > 1:
|
|
86
|
+
pid = parent[pid]
|
|
87
|
+
chain.add(pid)
|
|
88
|
+
return chain
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def busy_processes(
|
|
92
|
+
regexes: Sequence[re.Pattern[str]], processes: Iterable[Process], exclude: set[int]
|
|
93
|
+
) -> list[str]:
|
|
94
|
+
"""Command lines that match a pattern, like ``pgrep -f``, leaving out ``exclude``."""
|
|
95
|
+
return [
|
|
96
|
+
command
|
|
97
|
+
for pid, _, command in processes
|
|
98
|
+
if pid not in exclude and any(r.search(command) for r in regexes)
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _gb(nbytes: int) -> str:
|
|
103
|
+
return f"{nbytes / GB:.1f} GB"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _no_room_reasons(
|
|
107
|
+
config: GuardConfig,
|
|
108
|
+
io: GuardIO,
|
|
109
|
+
regexes: Sequence[re.Pattern[str]],
|
|
110
|
+
quiet: dict[str, float | None],
|
|
111
|
+
) -> list[str]:
|
|
112
|
+
reasons = []
|
|
113
|
+
level = io.level()
|
|
114
|
+
swap = io.swap_used()
|
|
115
|
+
if level < config.wait_free:
|
|
116
|
+
reasons.append(f"memory available {level}% < {config.wait_free}%")
|
|
117
|
+
if config.swap_limit is not None and swap + config.swap_budget > config.swap_limit:
|
|
118
|
+
reasons.append(
|
|
119
|
+
f"swap {_gb(swap)} + budget {_gb(config.swap_budget)} > limit {_gb(config.swap_limit)}"
|
|
120
|
+
)
|
|
121
|
+
if regexes:
|
|
122
|
+
processes = io.processes()
|
|
123
|
+
busy = busy_processes(regexes, processes, ancestors(processes, os.getpid()))
|
|
124
|
+
if busy:
|
|
125
|
+
quiet["since"] = None
|
|
126
|
+
reasons.append("busy: " + "; ".join(command[:80] for command in busy[:3]))
|
|
127
|
+
elif config.quiet_for > 0:
|
|
128
|
+
# a chain of jobs leaves short gaps between them: on 2026-10-05 a run started in
|
|
129
|
+
# such a gap, and the next video job took swap past its budget within 20 s
|
|
130
|
+
if quiet["since"] is None:
|
|
131
|
+
quiet["since"] = io.clock()
|
|
132
|
+
quiet_so_far = io.clock() - quiet["since"]
|
|
133
|
+
if quiet_so_far < config.quiet_for:
|
|
134
|
+
reasons.append(f"quiet for {quiet_so_far:.0f} s of {config.quiet_for:.0f} s")
|
|
135
|
+
return reasons
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class _Signals:
|
|
139
|
+
"""While installed, SIGINT, SIGTERM and SIGHUP are recorded instead of acting at once, so the
|
|
140
|
+
guard can stop its command whatever it was doing when one arrived. A signal the guard was
|
|
141
|
+
started to ignore stays ignored. Handlers can only be installed from the main thread."""
|
|
142
|
+
|
|
143
|
+
HANDLED = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)
|
|
144
|
+
|
|
145
|
+
def __init__(self) -> None:
|
|
146
|
+
self.first: int | None = None
|
|
147
|
+
self.count = 0
|
|
148
|
+
self._previous: dict[int, Any] = {}
|
|
149
|
+
|
|
150
|
+
def __enter__(self) -> _Signals:
|
|
151
|
+
if threading.current_thread() is threading.main_thread():
|
|
152
|
+
for signum in self.HANDLED:
|
|
153
|
+
if signal.getsignal(signum) is not signal.SIG_IGN:
|
|
154
|
+
self._previous[signum] = signal.signal(signum, self._record)
|
|
155
|
+
return self
|
|
156
|
+
|
|
157
|
+
def __exit__(self, *exc: object) -> None:
|
|
158
|
+
for signum, handler in self._previous.items():
|
|
159
|
+
signal.signal(signum, handler)
|
|
160
|
+
|
|
161
|
+
def _record(self, signum: int, frame: FrameType | None) -> None:
|
|
162
|
+
if self.first is None:
|
|
163
|
+
self.first = signum
|
|
164
|
+
self.count += 1
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
_SLICE = 0.2 # seconds: how long the guard can take to notice a signal
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _nap(io: GuardIO, seconds: float, signals: _Signals | None) -> None:
|
|
171
|
+
slices = max(1, math.ceil(seconds / _SLICE))
|
|
172
|
+
for _ in range(slices):
|
|
173
|
+
if signals is not None and signals.count:
|
|
174
|
+
return
|
|
175
|
+
io.sleep(seconds / slices)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def wait_for_room(config: GuardConfig, io: GuardIO, signals: _Signals | None = None) -> bool:
|
|
179
|
+
"""True once there is room; False after ``start_timeout``, or as soon as a signal arrives."""
|
|
180
|
+
regexes = [re.compile(pattern) for pattern in config.busy]
|
|
181
|
+
deadline = io.clock() + config.start_timeout
|
|
182
|
+
last = ""
|
|
183
|
+
quiet: dict[str, float | None] = {"since": None}
|
|
184
|
+
while True:
|
|
185
|
+
if signals is not None and signals.count:
|
|
186
|
+
return False
|
|
187
|
+
reasons = _no_room_reasons(config, io, regexes, quiet)
|
|
188
|
+
if not reasons:
|
|
189
|
+
return True
|
|
190
|
+
reason = ", ".join(reasons)
|
|
191
|
+
kind = reason.split(" for ")[0] if reason.startswith("quiet") else reason
|
|
192
|
+
if kind != last:
|
|
193
|
+
io.log(f"guard: waiting ({reason})")
|
|
194
|
+
last = kind
|
|
195
|
+
if io.clock() >= deadline:
|
|
196
|
+
io.log(f"guard: no room after {config.start_timeout:.0f} s ({reason})")
|
|
197
|
+
return False
|
|
198
|
+
_nap(io, config.poll, signals)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _group_alive(pgid: int) -> bool:
|
|
202
|
+
try:
|
|
203
|
+
os.killpg(pgid, 0)
|
|
204
|
+
except ProcessLookupError:
|
|
205
|
+
return False
|
|
206
|
+
except PermissionError:
|
|
207
|
+
return True
|
|
208
|
+
return True
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _stop_group(
|
|
212
|
+
proc: subprocess.Popen[bytes], grace: float, signals: _Signals | None = None
|
|
213
|
+
) -> None:
|
|
214
|
+
"""SIGTERM to the command's group, then SIGKILL to whatever is left of it once ``grace``
|
|
215
|
+
seconds have passed, or at once if the guard gets another signal meanwhile.
|
|
216
|
+
|
|
217
|
+
The grace is for the whole group: a shell that exits at once does not cut short the cleanup
|
|
218
|
+
of the job it started.
|
|
219
|
+
"""
|
|
220
|
+
pgid = proc.pid
|
|
221
|
+
seen = signals.count if signals is not None else 0
|
|
222
|
+
try:
|
|
223
|
+
os.killpg(pgid, signal.SIGTERM)
|
|
224
|
+
except ProcessLookupError:
|
|
225
|
+
proc.wait()
|
|
226
|
+
return
|
|
227
|
+
deadline = time.monotonic() + grace
|
|
228
|
+
while time.monotonic() < deadline:
|
|
229
|
+
proc.poll() # reap the leader, or it stays in the group as a zombie
|
|
230
|
+
if not _group_alive(pgid):
|
|
231
|
+
break
|
|
232
|
+
if signals is not None and signals.count > seen:
|
|
233
|
+
break
|
|
234
|
+
time.sleep(0.05)
|
|
235
|
+
try:
|
|
236
|
+
os.killpg(pgid, signal.SIGKILL)
|
|
237
|
+
except ProcessLookupError:
|
|
238
|
+
pass
|
|
239
|
+
proc.wait()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _wait(proc: subprocess.Popen[bytes], seconds: float, signals: _Signals) -> int | None:
|
|
243
|
+
"""The command's exit code if it ends within ``seconds``; None if it is still running or a
|
|
244
|
+
signal arrived."""
|
|
245
|
+
deadline = time.monotonic() + seconds
|
|
246
|
+
while True:
|
|
247
|
+
remaining = deadline - time.monotonic()
|
|
248
|
+
try:
|
|
249
|
+
return proc.wait(timeout=max(0.0, min(_SLICE, remaining)))
|
|
250
|
+
except subprocess.TimeoutExpired:
|
|
251
|
+
if signals.count or remaining <= _SLICE:
|
|
252
|
+
return None
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _exit_code(code: int) -> int:
|
|
256
|
+
"""A command ended by a signal reports -N; a shell reports 128+N."""
|
|
257
|
+
return 128 - code if code < 0 else code
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _signal_name(signum: int) -> str:
|
|
261
|
+
try:
|
|
262
|
+
return signal.Signals(signum).name
|
|
263
|
+
except ValueError:
|
|
264
|
+
return f"signal {signum}"
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def run_guarded(cmd: Sequence[str], config: GuardConfig, io: GuardIO | None = None) -> int:
|
|
268
|
+
io = io or system_io()
|
|
269
|
+
with _Signals() as signals:
|
|
270
|
+
room = wait_for_room(config, io, signals)
|
|
271
|
+
if signals.first is not None:
|
|
272
|
+
io.log(f"guard: got {_signal_name(signals.first)} before starting the command")
|
|
273
|
+
return 128 + signals.first
|
|
274
|
+
if not room:
|
|
275
|
+
return EXIT_NO_ROOM
|
|
276
|
+
swap_at_start = io.swap_used()
|
|
277
|
+
limit = swap_at_start + config.swap_budget
|
|
278
|
+
if config.swap_limit is not None:
|
|
279
|
+
limit = min(limit, config.swap_limit)
|
|
280
|
+
io.log(
|
|
281
|
+
f"guard: start, swap {_gb(swap_at_start)}, stop above {_gb(limit)} "
|
|
282
|
+
f"or under {config.kill_free}% available twice"
|
|
283
|
+
)
|
|
284
|
+
try:
|
|
285
|
+
proc = subprocess.Popen(list(cmd), start_new_session=True)
|
|
286
|
+
except OSError as error:
|
|
287
|
+
io.log(f"guard: cannot start {cmd[0]!r}: {error.strerror or error}")
|
|
288
|
+
return EXIT_CANNOT_START
|
|
289
|
+
try:
|
|
290
|
+
return _watch(proc, config, io, signals, limit)
|
|
291
|
+
except BaseException:
|
|
292
|
+
# the command cannot be watched any more, so it does not keep running unwatched
|
|
293
|
+
_stop_group(proc, config.grace, signals)
|
|
294
|
+
raise
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _watch(
|
|
298
|
+
proc: subprocess.Popen[bytes],
|
|
299
|
+
config: GuardConfig,
|
|
300
|
+
io: GuardIO,
|
|
301
|
+
signals: _Signals,
|
|
302
|
+
limit: int,
|
|
303
|
+
) -> int:
|
|
304
|
+
low = 0
|
|
305
|
+
while True:
|
|
306
|
+
if signals.first is not None:
|
|
307
|
+
io.log(
|
|
308
|
+
f"guard: got {_signal_name(signals.first)}, stopping pid {proc.pid} and its group"
|
|
309
|
+
)
|
|
310
|
+
_stop_group(proc, config.grace, signals)
|
|
311
|
+
return 128 + signals.first
|
|
312
|
+
code = _wait(proc, config.poll, signals)
|
|
313
|
+
if code is not None:
|
|
314
|
+
io.log(f"guard: exit {_exit_code(code)}")
|
|
315
|
+
return _exit_code(code)
|
|
316
|
+
if signals.first is not None:
|
|
317
|
+
continue
|
|
318
|
+
level = io.level()
|
|
319
|
+
swap = io.swap_used()
|
|
320
|
+
low = low + 1 if level < config.kill_free else 0
|
|
321
|
+
if swap > limit or low >= 2:
|
|
322
|
+
if swap > limit:
|
|
323
|
+
why = f"swap {_gb(swap)} > {_gb(limit)}"
|
|
324
|
+
else:
|
|
325
|
+
why = f"memory available {level}% < {config.kill_free}% twice"
|
|
326
|
+
io.log(f"guard: killing pid {proc.pid} and its group ({why})")
|
|
327
|
+
_stop_group(proc, config.grace, signals)
|
|
328
|
+
return EXIT_KILLED
|
stageload/hooks.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Run callbacks around one method of one object."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def on_call(
|
|
11
|
+
obj: Any,
|
|
12
|
+
method: str,
|
|
13
|
+
*,
|
|
14
|
+
before: Callable[..., None] | None = None,
|
|
15
|
+
after: Callable[..., None] | None = None,
|
|
16
|
+
) -> Callable[[], None]:
|
|
17
|
+
"""Wrap ``obj.method`` on this instance only; return a function that removes the wrapper.
|
|
18
|
+
|
|
19
|
+
``before(*args, **kwargs)`` runs before the call, ``after(result, *args, **kwargs)`` after
|
|
20
|
+
it. Wrappers stack; remove them in reverse order.
|
|
21
|
+
"""
|
|
22
|
+
if before is None and after is None:
|
|
23
|
+
raise ValueError("on_call needs a before or an after callback")
|
|
24
|
+
original = getattr(obj, method)
|
|
25
|
+
had_own = method in vars(obj)
|
|
26
|
+
previous = vars(obj).get(method)
|
|
27
|
+
|
|
28
|
+
@functools.wraps(original)
|
|
29
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
30
|
+
if before is not None:
|
|
31
|
+
before(*args, **kwargs)
|
|
32
|
+
result = original(*args, **kwargs)
|
|
33
|
+
if after is not None:
|
|
34
|
+
after(result, *args, **kwargs)
|
|
35
|
+
return result
|
|
36
|
+
|
|
37
|
+
setattr(obj, method, wrapper)
|
|
38
|
+
|
|
39
|
+
def uninstall() -> None:
|
|
40
|
+
if vars(obj).get(method) is not wrapper:
|
|
41
|
+
raise RuntimeError(
|
|
42
|
+
f"another wrapper was installed on {method!r} after this one; "
|
|
43
|
+
"uninstall them in reverse order"
|
|
44
|
+
)
|
|
45
|
+
if had_own:
|
|
46
|
+
setattr(obj, method, previous)
|
|
47
|
+
else:
|
|
48
|
+
delattr(obj, method)
|
|
49
|
+
|
|
50
|
+
return uninstall
|