hyperun 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hyperun-0.2.2 → hyperun-0.2.4}/PKG-INFO +1 -1
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun/cli.py +75 -3
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun/client.py +21 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/PKG-INFO +1 -1
- {hyperun-0.2.2 → hyperun-0.2.4}/pyproject.toml +1 -1
- {hyperun-0.2.2 → hyperun-0.2.4}/tests/test_cli.py +50 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/LICENSE +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/README.md +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun/__init__.py +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun/browser_login.py +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun/config.py +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/SOURCES.txt +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/dependency_links.txt +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/entry_points.txt +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/requires.txt +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/hyperun.egg-info/top_level.txt +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/setup.cfg +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/tests/test_browser_login.py +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/tests/test_client.py +0 -0
- {hyperun-0.2.2 → hyperun-0.2.4}/tests/test_config.py +0 -0
|
@@ -116,9 +116,10 @@ def job_arguments() -> argparse.ArgumentParser:
|
|
|
116
116
|
"--vendor", action="append", metavar="NAME",
|
|
117
117
|
help="who the machine may be bought from. Repeat it to allow several; "
|
|
118
118
|
"omit it entirely for no restriction, which is what every job did before "
|
|
119
|
-
"this flag existed. aws and
|
|
120
|
-
"
|
|
121
|
-
"
|
|
119
|
+
"this flag existed. aws, runpod and shadeform can actually run a job, "
|
|
120
|
+
"and `hyperun estimate` prices all three and names the cheapest; gcp, "
|
|
121
|
+
"azure, lambda and nebius can only be PRICED, so name one of those only "
|
|
122
|
+
"with --placement-mode compare.",
|
|
122
123
|
)
|
|
123
124
|
# DDPSRUN-REGIONS. Missing until 2026-09-08, so every job this CLI submitted
|
|
124
125
|
# ran in the operator's one default region and there was no way to say
|
|
@@ -279,6 +280,34 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
279
280
|
"the job's result path are NOT deleted.",
|
|
280
281
|
)
|
|
281
282
|
cancel.add_argument("job_id", help="the id `submit` printed")
|
|
283
|
+
|
|
284
|
+
# HYPERUN-JOB-STOP. `pause` and `resume` are two verbs over one boolean, because that
|
|
285
|
+
# is how a person thinks about it -- `stop --undo` would be one verb and two readings.
|
|
286
|
+
pause = sub.add_parser(
|
|
287
|
+
"pause",
|
|
288
|
+
help="pause a running job, KEEPING its machine",
|
|
289
|
+
description="Asks for the job's machine to be paused. Compute billing stops; the "
|
|
290
|
+
"disk keeps costing, and on RunPod a stopped volume costs MORE than a running one "
|
|
291
|
+
"($0.20/GB-month against $0.10). Unlike `delete`, the job stays in the list and "
|
|
292
|
+
"everything it has produced stays on the machine.\n\n"
|
|
293
|
+
"IT ASKS. The pause happens seconds to a minute later, and on some vendors it "
|
|
294
|
+
"cannot happen at all: Shadeform has no stop API, a spot instance has no stopped "
|
|
295
|
+
"state, and a RunPod pod with no volume disk would lose everything it has "
|
|
296
|
+
"produced -- so the driver refuses rather than destroying the run, and says why "
|
|
297
|
+
"in the job's log. `hyperun status` shows the phase reaching Stopped when it "
|
|
298
|
+
"actually did.",
|
|
299
|
+
)
|
|
300
|
+
pause.add_argument("job_id", help="the id `submit` printed")
|
|
301
|
+
|
|
302
|
+
resume = sub.add_parser(
|
|
303
|
+
"resume",
|
|
304
|
+
help="let a paused job run again",
|
|
305
|
+
description="Asks for a paused job to run again on the machine it kept. If the "
|
|
306
|
+
"vendor cannot give the machine back -- RunPod says plainly that a restarted pod "
|
|
307
|
+
"may be allocated zero GPUs -- the job STAYS paused and says so, rather than going "
|
|
308
|
+
"to buy a different machine, which would be a recovery wearing a resume's name.",
|
|
309
|
+
)
|
|
310
|
+
resume.add_argument("job_id", help="the id `submit` printed")
|
|
282
311
|
cancel.add_argument(
|
|
283
312
|
"--yes", "-y", action="store_true",
|
|
284
313
|
help="skip the confirmation. For scripts, which have nobody to answer it.",
|
|
@@ -1076,6 +1105,47 @@ def cmd_delete(args: argparse.Namespace) -> int:
|
|
|
1076
1105
|
return EXIT_OK
|
|
1077
1106
|
|
|
1078
1107
|
|
|
1108
|
+
def _pause_or_resume(args: argparse.Namespace, stopped: bool) -> int:
|
|
1109
|
+
"""One boolean, two verbs. HYPERUN-JOB-STOP.
|
|
1110
|
+
|
|
1111
|
+
★ IT PRINTS WHAT WAS ASKED AND WHAT HAS HAPPENED, and they are different things. The
|
|
1112
|
+
request lands immediately; the machine pauses seconds to a minute later, because the
|
|
1113
|
+
controller writes an annotation and the kubelet copies it into the driver pod on its
|
|
1114
|
+
own period. On the vendors where a pause cannot happen it never will, and the reason
|
|
1115
|
+
reaches the job's log rather than this reply -- only the driver, which holds the cloud
|
|
1116
|
+
credential, knows which of those it is.
|
|
1117
|
+
"""
|
|
1118
|
+
verb = "pause" if stopped else "resume"
|
|
1119
|
+
try:
|
|
1120
|
+
job = client_from_config().set_stopped(args.job_id, stopped)
|
|
1121
|
+
except ServerError as exc:
|
|
1122
|
+
print(str(exc), file=sys.stderr)
|
|
1123
|
+
return EXIT_SERVER
|
|
1124
|
+
phase = job.get("phase", "?")
|
|
1125
|
+
print(f"asked to {verb} {args.job_id}; it is {phase} right now")
|
|
1126
|
+
if stopped:
|
|
1127
|
+
print(" the machine pauses in up to a minute. `hyperun status "
|
|
1128
|
+
f"{args.job_id}` shows Stopped when it actually has, and the job's log says "
|
|
1129
|
+
"so if this vendor cannot pause it.")
|
|
1130
|
+
print(" the disk keeps costing while it is paused, and the orphan sweep stops "
|
|
1131
|
+
"protecting a job left paused for 7 days.")
|
|
1132
|
+
else:
|
|
1133
|
+
print(f" `hyperun status {args.job_id}` shows it Running again once the machine "
|
|
1134
|
+
"comes back. If the vendor has no GPU to give, it STAYS Stopped and says so "
|
|
1135
|
+
"rather than buying a different machine.")
|
|
1136
|
+
return EXIT_OK
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def cmd_pause(args: argparse.Namespace) -> int:
|
|
1140
|
+
"""Pause a running job, keeping its machine."""
|
|
1141
|
+
return _pause_or_resume(args, True)
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def cmd_resume(args: argparse.Namespace) -> int:
|
|
1145
|
+
"""Let a paused job run again."""
|
|
1146
|
+
return _pause_or_resume(args, False)
|
|
1147
|
+
|
|
1148
|
+
|
|
1079
1149
|
def bar(percent: float, width: int = 20) -> str:
|
|
1080
1150
|
"""Draw a progress bar.
|
|
1081
1151
|
|
|
@@ -1382,6 +1452,8 @@ COMMANDS = {
|
|
|
1382
1452
|
"delete": cmd_delete,
|
|
1383
1453
|
# The old spelling, kept as an alias -- see the comment on the parser.
|
|
1384
1454
|
"cancel": cmd_delete,
|
|
1455
|
+
"pause": cmd_pause,
|
|
1456
|
+
"resume": cmd_resume,
|
|
1385
1457
|
"logout": cmd_logout,
|
|
1386
1458
|
"explain": cmd_explain,
|
|
1387
1459
|
"schema": cmd_schema,
|
|
@@ -132,6 +132,27 @@ class Client:
|
|
|
132
132
|
"""
|
|
133
133
|
self._call("DELETE", f"/v1/jobs/{job_id}")
|
|
134
134
|
|
|
135
|
+
def set_stopped(self, job_id: str, stopped: bool) -> dict:
|
|
136
|
+
"""Ask for a job to be PAUSED, keeping its machine -- or to run again.
|
|
137
|
+
|
|
138
|
+
★ IT ASKS. The machine pauses seconds to a minute later, and on some vendors it
|
|
139
|
+
cannot pause at all. What comes back carries both: `stopped` is the request,
|
|
140
|
+
`phase` is what has actually happened.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
job_id: an id the server issued.
|
|
144
|
+
stopped: True to pause, False to resume.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
The job as it now reads.
|
|
148
|
+
|
|
149
|
+
Raises:
|
|
150
|
+
ServerError: 404 for no such job of yours, 409 for one that has finished,
|
|
151
|
+
502 when the cluster could not be reached.
|
|
152
|
+
"""
|
|
153
|
+
suffix = "" if stopped else "?resume=true"
|
|
154
|
+
return self._call("POST", f"/v1/jobs/{job_id}/stop{suffix}") or {}
|
|
155
|
+
|
|
135
156
|
def status(self, job_id: str) -> dict[str, Any]:
|
|
136
157
|
"""Read one job's state."""
|
|
137
158
|
return self._call("GET", f"/v1/jobs/{job_id}").json()
|
|
@@ -34,7 +34,7 @@ build-backend = "setuptools.build_meta"
|
|
|
34
34
|
|
|
35
35
|
[project]
|
|
36
36
|
name = "hyperun"
|
|
37
|
-
version = "0.2.
|
|
37
|
+
version = "0.2.4"
|
|
38
38
|
description = "Submit a GPU job and get results back. No kubectl, no cloud account."
|
|
39
39
|
requires-python = ">=3.9"
|
|
40
40
|
dependencies = ["requests>=2.31", "PyYAML>=6.0"]
|
|
@@ -1222,3 +1222,53 @@ def test_a_failure_that_is_not_a_lost_session_is_not_retried(fake, monkeypatch,
|
|
|
1222
1222
|
assert run(["shell", "job-a8acdef80a07"]) == 0
|
|
1223
1223
|
assert calls["n"] == 1, "retrying an unrelated failure would double every bad command"
|
|
1224
1224
|
assert "did not answer in JSON" in capsys.readouterr().err
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
# ------------------------------------------------------------- HYPERUN-JOB-STOP
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def test_pause_and_resume_are_two_verbs_over_one_boolean(monkeypatch, capsys):
|
|
1231
|
+
"""★ `pause` AND `resume` RATHER THAN `stop --undo`. One verb with a flag is one verb
|
|
1232
|
+
and two readings; a person asking to pause a job and a person asking to resume one are
|
|
1233
|
+
doing opposite things and should type opposite words."""
|
|
1234
|
+
calls = []
|
|
1235
|
+
|
|
1236
|
+
class FakeClient:
|
|
1237
|
+
def set_stopped(self, job_id, stopped):
|
|
1238
|
+
calls.append((job_id, stopped))
|
|
1239
|
+
return {"phase": "Stopped" if stopped else "Running", "stopped": stopped}
|
|
1240
|
+
|
|
1241
|
+
monkeypatch.setattr(cli, "client_from_config", lambda: FakeClient())
|
|
1242
|
+
|
|
1243
|
+
assert cli.main(["pause", "job-abc123abc123"]) == 0
|
|
1244
|
+
assert cli.main(["resume", "job-abc123abc123"]) == 0
|
|
1245
|
+
assert calls == [("job-abc123abc123", True), ("job-abc123abc123", False)]
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
def test_pause_says_it_asked_rather_than_that_it_stopped(monkeypatch, capsys):
|
|
1249
|
+
"""★★ THE WORD MATTERS. The machine pauses up to a minute later, and on some vendors it
|
|
1250
|
+
never does. "paused" here would tell somebody a $6/hour machine had stopped billing when
|
|
1251
|
+
it had not."""
|
|
1252
|
+
class FakeClient:
|
|
1253
|
+
def set_stopped(self, job_id, stopped):
|
|
1254
|
+
return {"phase": "Running", "stopped": True}
|
|
1255
|
+
|
|
1256
|
+
monkeypatch.setattr(cli, "client_from_config", lambda: FakeClient())
|
|
1257
|
+
cli.main(["pause", "job-abc123abc123"])
|
|
1258
|
+
out = capsys.readouterr().out
|
|
1259
|
+
assert "asked to pause" in out
|
|
1260
|
+
assert "Running right now" in out, "the phase has to be the real one, not the request"
|
|
1261
|
+
# And the two things a person needs to know next.
|
|
1262
|
+
assert "cannot pause" in out
|
|
1263
|
+
assert "7 days" in out, "a job left paused stops being protected, and that has to be said"
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
def test_a_refusal_from_the_server_is_not_swallowed(monkeypatch, capsys):
|
|
1267
|
+
class FakeClient:
|
|
1268
|
+
def set_stopped(self, job_id, stopped):
|
|
1269
|
+
raise cli.ServerError("this job has finished (Succeeded)")
|
|
1270
|
+
|
|
1271
|
+
monkeypatch.setattr(cli, "client_from_config", lambda: FakeClient())
|
|
1272
|
+
assert cli.main(["pause", "job-abc123abc123"]) != 0
|
|
1273
|
+
assert "Succeeded" in capsys.readouterr().err
|
|
1274
|
+
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|