hyperun 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hyperun/__init__.py +24 -0
- hyperun/browser_login.py +281 -0
- hyperun/cli.py +1202 -0
- hyperun/client.py +280 -0
- hyperun/config.py +218 -0
- hyperun-0.2.0.dist-info/METADATA +183 -0
- hyperun-0.2.0.dist-info/RECORD +11 -0
- hyperun-0.2.0.dist-info/WHEEL +5 -0
- hyperun-0.2.0.dist-info/entry_points.txt +2 -0
- hyperun-0.2.0.dist-info/licenses/LICENSE +202 -0
- hyperun-0.2.0.dist-info/top_level.txt +1 -0
hyperun/client.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""The HTTP calls, and nothing else.
|
|
2
|
+
|
|
3
|
+
END-TO-END FLOW of this file:
|
|
4
|
+
|
|
5
|
+
1. `cli.py` builds a `Client` from the credentials `config.load()` returned.
|
|
6
|
+
2. Every method turns one CLI command into one HTTP request against the
|
|
7
|
+
gateway server's routes.
|
|
8
|
+
3. A non-2xx response becomes a `ServerError` carrying the server's own
|
|
9
|
+
`detail` string, because that string is written for a human — the server
|
|
10
|
+
passes the CRD's validation messages through verbatim for this reason.
|
|
11
|
+
4. `logs()` is a generator so that `--follow` prints lines as they arrive
|
|
12
|
+
rather than buffering a training run's entire output.
|
|
13
|
+
|
|
14
|
+
WHY THIS IS SEPARATE FROM cli.py. `cli.py` decides what to print; this decides
|
|
15
|
+
what to ask for. Keeping them apart means the tests here can use a fake HTTP
|
|
16
|
+
layer and the tests there can use a fake client, and neither needs a server.
|
|
17
|
+
|
|
18
|
+
WHY THERE IS NO RETRY. A submit that timed out may or may not have created a
|
|
19
|
+
job, and retrying it would create a second one that also rents a GPU. Until the
|
|
20
|
+
API takes an idempotency key, the honest thing is to fail and say so.
|
|
21
|
+
|
|
22
|
+
Grep anchor: DDPSRUN-CLI-CLIENT
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from typing import Any
|
|
28
|
+
from urllib.parse import quote
|
|
29
|
+
|
|
30
|
+
import requests
|
|
31
|
+
|
|
32
|
+
# A submit does real work (it creates an object and starts a rental), so it gets
|
|
33
|
+
# a longer patience than a status read. Neither is the job's own runtime — the
|
|
34
|
+
# server answers immediately in both cases and these only cover the network.
|
|
35
|
+
SUBMIT_TIMEOUT = 30
|
|
36
|
+
READ_TIMEOUT = 15
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ServerError(Exception):
|
|
40
|
+
"""The server refused, or could not be reached. `cli.py` prints and exits 1."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Client:
|
|
44
|
+
"""One user's connection to one gateway server."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, server: str, token: str, session: Any | None = None) -> None:
|
|
47
|
+
"""
|
|
48
|
+
Args:
|
|
49
|
+
server: base URL, no trailing slash.
|
|
50
|
+
token: the bearer token.
|
|
51
|
+
session: an object with `.request`, for tests. Defaults to a real
|
|
52
|
+
`requests.Session`.
|
|
53
|
+
"""
|
|
54
|
+
self._server = server.rstrip("/")
|
|
55
|
+
self._token = token
|
|
56
|
+
self._session = session or requests.Session()
|
|
57
|
+
|
|
58
|
+
def _headers(self, authenticated: bool = True) -> dict[str, str]:
|
|
59
|
+
headers = {"Accept": "application/json"}
|
|
60
|
+
if authenticated:
|
|
61
|
+
headers["Authorization"] = f"Bearer {self._token}"
|
|
62
|
+
return headers
|
|
63
|
+
|
|
64
|
+
def _call(
|
|
65
|
+
self,
|
|
66
|
+
method: str,
|
|
67
|
+
path: str,
|
|
68
|
+
*,
|
|
69
|
+
json_body: dict[str, Any] | None = None,
|
|
70
|
+
timeout: int = READ_TIMEOUT,
|
|
71
|
+
authenticated: bool = True,
|
|
72
|
+
stream: bool = False,
|
|
73
|
+
) -> requests.Response:
|
|
74
|
+
"""Make one request and turn any failure into a `ServerError`.
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
ServerError: the connection failed, timed out, or the response was
|
|
78
|
+
not 2xx. The message is the server's `detail` when there is one.
|
|
79
|
+
"""
|
|
80
|
+
try:
|
|
81
|
+
response = self._session.request(
|
|
82
|
+
method,
|
|
83
|
+
f"{self._server}{path}",
|
|
84
|
+
json=json_body,
|
|
85
|
+
headers=self._headers(authenticated),
|
|
86
|
+
timeout=timeout,
|
|
87
|
+
stream=stream,
|
|
88
|
+
)
|
|
89
|
+
except requests.RequestException as exc:
|
|
90
|
+
raise ServerError(f"cannot reach {self._server}: {exc}") from exc
|
|
91
|
+
|
|
92
|
+
if response.status_code >= 400:
|
|
93
|
+
raise ServerError(_detail(response))
|
|
94
|
+
return response
|
|
95
|
+
|
|
96
|
+
def explain(self) -> str:
|
|
97
|
+
"""Fetch the server's own description of itself. No token needed."""
|
|
98
|
+
return self._call("GET", "/v1/explain", authenticated=False).text
|
|
99
|
+
|
|
100
|
+
def schema(self) -> dict[str, Any]:
|
|
101
|
+
"""Fetch the JSON Schema of a submit request. No token needed."""
|
|
102
|
+
return self._call("GET", "/v1/schema", authenticated=False).json()
|
|
103
|
+
|
|
104
|
+
def estimate(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
105
|
+
"""Ask how long a job will take and what it will cost. Submits nothing."""
|
|
106
|
+
return self._call("POST", "/v1/estimate", json_body=body).json()
|
|
107
|
+
|
|
108
|
+
def validate(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
109
|
+
"""Ask what is wrong with a job. Submits nothing."""
|
|
110
|
+
return self._call("POST", "/v1/validate", json_body=body).json()
|
|
111
|
+
|
|
112
|
+
def submit(self, body: dict[str, Any]) -> dict[str, Any]:
|
|
113
|
+
"""Submit a job.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
body: the submit request, already assembled by `cli.py`.
|
|
117
|
+
|
|
118
|
+
Returns:
|
|
119
|
+
`{job_id, name, result_path}`.
|
|
120
|
+
"""
|
|
121
|
+
return self._call("POST", "/v1/jobs", json_body=body, timeout=SUBMIT_TIMEOUT).json()
|
|
122
|
+
|
|
123
|
+
def cancel(self, job_id: str) -> None:
|
|
124
|
+
"""Stop a job and take it off the list.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
job_id: an id the server issued.
|
|
128
|
+
|
|
129
|
+
Raises:
|
|
130
|
+
ServerError: 404 when there is no such job of yours, 502 when the
|
|
131
|
+
cluster could not be reached.
|
|
132
|
+
"""
|
|
133
|
+
self._call("DELETE", f"/v1/jobs/{job_id}")
|
|
134
|
+
|
|
135
|
+
def status(self, job_id: str) -> dict[str, Any]:
|
|
136
|
+
"""Read one job's state."""
|
|
137
|
+
return self._call("GET", f"/v1/jobs/{job_id}").json()
|
|
138
|
+
|
|
139
|
+
def secrets(self) -> dict[str, Any]:
|
|
140
|
+
"""The words this deployment accepts in `secrets`.
|
|
141
|
+
|
|
142
|
+
Names only — never values, and never which Kubernetes Secret holds
|
|
143
|
+
them. Storing a new one is an operator's job in the cluster; this
|
|
144
|
+
answers the question a submitter actually has, which is "what may I
|
|
145
|
+
write".
|
|
146
|
+
"""
|
|
147
|
+
return self._call("GET", "/v1/secrets").json()
|
|
148
|
+
|
|
149
|
+
def put_secret(self, name: str, value: str,
|
|
150
|
+
expires_at: str | None = None) -> dict[str, Any]:
|
|
151
|
+
"""Store one value under `name`, for this caller's own namespace.
|
|
152
|
+
|
|
153
|
+
DDPSRUN-USER-SECRET. The one call in this file that carries a secret.
|
|
154
|
+
It goes in the BODY and never in the path or a query string: a URL is
|
|
155
|
+
logged by every hop that touches it, and the server's own routes are
|
|
156
|
+
forbidden from putting personal data in a query string for the same
|
|
157
|
+
reason. The value is not returned by this or any other call.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
name: the environment variable name, e.g. `HF_TOKEN`.
|
|
161
|
+
value: the secret, already read from a file or stdin by `cli.py`.
|
|
162
|
+
expires_at: when it stops working, ISO-8601. Omitted for a value
|
|
163
|
+
that does not expire. DDPSRUN-SECRET-EXPIRY.
|
|
164
|
+
|
|
165
|
+
Returns:
|
|
166
|
+
`{name, namespace, created}`.
|
|
167
|
+
|
|
168
|
+
Raises:
|
|
169
|
+
ServerError: 400 for an unusable name or value, 403 for someone
|
|
170
|
+
else's namespace, 409 for a name the deployment already binds,
|
|
171
|
+
502 when the cluster refused. The server's own message says
|
|
172
|
+
which.
|
|
173
|
+
"""
|
|
174
|
+
body: dict[str, Any] = {"value": value}
|
|
175
|
+
if expires_at:
|
|
176
|
+
body["expires_at"] = expires_at
|
|
177
|
+
return self._call(
|
|
178
|
+
"PUT", f"/v1/secrets/{quote(name)}", json_body=body
|
|
179
|
+
).json()
|
|
180
|
+
|
|
181
|
+
def delete_secret(self, name: str) -> None:
|
|
182
|
+
"""Forget one name this caller's namespace registered.
|
|
183
|
+
|
|
184
|
+
Raises:
|
|
185
|
+
ServerError: 404 when the name is not registered here, which is
|
|
186
|
+
also the answer for an operator's binding -- those are not
|
|
187
|
+
this caller's to remove.
|
|
188
|
+
"""
|
|
189
|
+
self._call("DELETE", f"/v1/secrets/{quote(name)}")
|
|
190
|
+
|
|
191
|
+
def stats(self) -> dict[str, Any]:
|
|
192
|
+
"""Read this caller's team figures. Aggregate only."""
|
|
193
|
+
return self._call("GET", "/v1/stats").json()
|
|
194
|
+
|
|
195
|
+
def exec_in_job(
|
|
196
|
+
self, job: str, command: str, slot: int = 0, timeout_seconds: int = 20
|
|
197
|
+
) -> dict[str, Any]:
|
|
198
|
+
"""Run one shell line inside a running job's workload container.
|
|
199
|
+
|
|
200
|
+
The server relays it through the job's driver pod onto the rented
|
|
201
|
+
machine and brings the exit code back like ssh would. One command per
|
|
202
|
+
request — there is no held-open terminal behind a Lambda.
|
|
203
|
+
|
|
204
|
+
Args:
|
|
205
|
+
job: the job id, or the PacsJob's Kubernetes name.
|
|
206
|
+
command: one shell line, run as `sh -lc <command>`.
|
|
207
|
+
slot: which pod of a parallel job.
|
|
208
|
+
timeout_seconds: server-side wait, capped at 25 by the server.
|
|
209
|
+
"""
|
|
210
|
+
return self._call(
|
|
211
|
+
"POST",
|
|
212
|
+
f"/v1/jobs/{job}/exec",
|
|
213
|
+
json_body={
|
|
214
|
+
"command": command,
|
|
215
|
+
"slot": slot,
|
|
216
|
+
"timeout_seconds": timeout_seconds,
|
|
217
|
+
},
|
|
218
|
+
# The server may hold the request for timeout_seconds before
|
|
219
|
+
# answering; the read timeout has to outlive that on purpose.
|
|
220
|
+
timeout=timeout_seconds + 10,
|
|
221
|
+
).json()
|
|
222
|
+
|
|
223
|
+
def metrics(self, job_id: str, window_seconds: int = 3600) -> dict[str, Any]:
|
|
224
|
+
"""Read a job's GPU usage and training progress."""
|
|
225
|
+
return self._call(
|
|
226
|
+
"GET", f"/v1/jobs/{job_id}/metrics?window_seconds={window_seconds}"
|
|
227
|
+
).json()
|
|
228
|
+
|
|
229
|
+
def log_window(
|
|
230
|
+
self, job_id: str, since: str | None = None, window_seconds: int = 30
|
|
231
|
+
) -> dict[str, Any]:
|
|
232
|
+
"""Read one window of a job's output.
|
|
233
|
+
|
|
234
|
+
NOT A STREAM. The server cannot hold a connection open for a
|
|
235
|
+
thirty-hour job — a Lambda execution is capped at 15 minutes — so
|
|
236
|
+
`cmd_logs` calls this repeatedly and uses `last_timestamp` to skip what
|
|
237
|
+
it has already printed.
|
|
238
|
+
|
|
239
|
+
Args:
|
|
240
|
+
job_id: the id `submit` returned.
|
|
241
|
+
since: the previous call's `last_timestamp`, or None on the first.
|
|
242
|
+
window_seconds: how far back to read. Several times the interval
|
|
243
|
+
between calls, so a slow round trip does not lose lines.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
`{lines, last_timestamp, window_seconds}`.
|
|
247
|
+
"""
|
|
248
|
+
query = f"?window_seconds={window_seconds}"
|
|
249
|
+
if since:
|
|
250
|
+
query += f"&since={quote(since)}"
|
|
251
|
+
return self._call("GET", f"/v1/jobs/{job_id}/logs{query}").json()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _detail(response: requests.Response) -> str:
|
|
255
|
+
"""Pull the readable part out of an error response.
|
|
256
|
+
|
|
257
|
+
FastAPI puts it in `detail`, which is either a string (our own
|
|
258
|
+
`HTTPException`) or a list of field errors (pydantic's 422). Both are worth
|
|
259
|
+
showing; the raw JSON is not.
|
|
260
|
+
"""
|
|
261
|
+
try:
|
|
262
|
+
body = response.json()
|
|
263
|
+
except ValueError:
|
|
264
|
+
return f"server returned {response.status_code}"
|
|
265
|
+
|
|
266
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
267
|
+
if isinstance(detail, str):
|
|
268
|
+
return detail
|
|
269
|
+
if isinstance(detail, list):
|
|
270
|
+
parts = []
|
|
271
|
+
for item in detail:
|
|
272
|
+
if not isinstance(item, dict):
|
|
273
|
+
continue
|
|
274
|
+
# loc is like ["body", "gpu", "vram_gb"]; the first element is
|
|
275
|
+
# always "body" and says nothing.
|
|
276
|
+
where = ".".join(str(p) for p in item.get("loc", [])[1:]) or "request"
|
|
277
|
+
parts.append(f"{where}: {item.get('msg', 'invalid')}")
|
|
278
|
+
if parts:
|
|
279
|
+
return "; ".join(parts)
|
|
280
|
+
return f"server returned {response.status_code}"
|
hyperun/config.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Where the CLI keeps the server address and the user's token.
|
|
2
|
+
|
|
3
|
+
END-TO-END FLOW of this file:
|
|
4
|
+
|
|
5
|
+
1. `hyperun login` asks for a server URL and a token and calls `save()`.
|
|
6
|
+
2. `save()` writes them to `~/.config/hyperun/config.json` with mode 0600 and
|
|
7
|
+
creates the directory with mode 0700.
|
|
8
|
+
3. Every other command calls `load()`, which returns the same two values, or
|
|
9
|
+
raises `NotLoggedIn` with the exact command to run.
|
|
10
|
+
|
|
11
|
+
WHY A FILE AND NOT AN ENVIRONMENT VARIABLE. A token in an environment variable
|
|
12
|
+
is in the shell's history if it was ever exported on a command line, is
|
|
13
|
+
inherited by every process the user starts, and is gone on a new terminal. A
|
|
14
|
+
file survives, and 0600 is a boundary the operating system enforces.
|
|
15
|
+
`HYPERUN_TOKEN` is still honoured for scripts and CI, where a file is the wrong
|
|
16
|
+
shape — `load()` prefers it when it is set.
|
|
17
|
+
|
|
18
|
+
WHY XDG AND NOT ~/.hyperun. `XDG_CONFIG_HOME` is what a Linux user's backup and
|
|
19
|
+
dotfile tooling already knows about, and it falls back to `~/.config`, which is
|
|
20
|
+
where macOS users' tools look too.
|
|
21
|
+
|
|
22
|
+
★ THE COMMAND WAS `ddpsrun` UNTIL 2026-09-10, AND NOBODY IS LOGGED OUT BY THE
|
|
23
|
+
RENAME. Two pieces of state carried the old name and both are user-visible, so
|
|
24
|
+
both accept it still:
|
|
25
|
+
|
|
26
|
+
the directory `save()` writes `~/.config/hyperun/`, and `config_path()`
|
|
27
|
+
falls back to `~/.config/ddpsrun/config.json` when the new
|
|
28
|
+
one does not exist. A rename that silently logged everybody
|
|
29
|
+
out would be the same class of defect as the ones this file's
|
|
30
|
+
comments already record — the fix is one `exists()` check.
|
|
31
|
+
the env vars `HYPERUN_SERVER` / `HYPERUN_TOKEN` are read first, then
|
|
32
|
+
`DDPSRUN_SERVER` / `DDPSRUN_TOKEN`. CI files and shell
|
|
33
|
+
profiles already export the old pair.
|
|
34
|
+
|
|
35
|
+
Neither fallback is dated for removal here. Deleting them is a decision about
|
|
36
|
+
how long the old spelling has to keep working, which belongs to whoever knows
|
|
37
|
+
who still has it exported.
|
|
38
|
+
|
|
39
|
+
Grep anchor: DDPSRUN-CLI-CONFIG
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import json
|
|
45
|
+
import os
|
|
46
|
+
import stat
|
|
47
|
+
from dataclasses import dataclass
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
|
|
50
|
+
CONFIG_FILENAME = "config.json"
|
|
51
|
+
CONFIG_DIRNAME = "hyperun"
|
|
52
|
+
# The pre-2026-09-10 spelling of everything below. Read, never written.
|
|
53
|
+
LEGACY_CONFIG_DIRNAME = "ddpsrun"
|
|
54
|
+
SERVER_ENV = "HYPERUN_SERVER"
|
|
55
|
+
TOKEN_ENV = "HYPERUN_TOKEN"
|
|
56
|
+
LEGACY_SERVER_ENV = "DDPSRUN_SERVER"
|
|
57
|
+
LEGACY_TOKEN_ENV = "DDPSRUN_TOKEN"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class NotLoggedIn(Exception):
|
|
61
|
+
"""No credentials anywhere. `cli.py` prints this and exits 2."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Credentials:
|
|
66
|
+
"""Everything needed to reach the server.
|
|
67
|
+
|
|
68
|
+
Attributes:
|
|
69
|
+
server: the gateway URL.
|
|
70
|
+
token: what goes in `Authorization: Bearer`. Either a static token an
|
|
71
|
+
operator issued, or a Cognito id_token from `hyperun login`.
|
|
72
|
+
refresh_token: only set after a browser sign-in. An id_token lives an
|
|
73
|
+
hour; this buys a new one without opening a browser again, and is
|
|
74
|
+
why the file is written at mode 0600.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
server: str
|
|
78
|
+
token: str
|
|
79
|
+
refresh_token: str = ""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _xdg_base() -> Path:
|
|
83
|
+
"""The directory the config directory sits in.
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
`$XDG_CONFIG_HOME`, or `~/.config` when that is unset.
|
|
87
|
+
"""
|
|
88
|
+
return Path(os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config"))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def config_dir() -> Path:
|
|
92
|
+
"""Where a NEW config file is written.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
`$XDG_CONFIG_HOME/hyperun`, or `~/.config/hyperun` when that is unset.
|
|
96
|
+
Always the new name: `save()` migrates by writing here, and nothing
|
|
97
|
+
writes the old directory again.
|
|
98
|
+
"""
|
|
99
|
+
return _xdg_base() / CONFIG_DIRNAME
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def legacy_config_path() -> Path:
|
|
103
|
+
"""The file the command wrote while it was called `ddpsrun`.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
`<xdg base>/ddpsrun/config.json`. It may not exist, which is the normal
|
|
107
|
+
case for anybody who first logged in after the rename.
|
|
108
|
+
"""
|
|
109
|
+
return _xdg_base() / LEGACY_CONFIG_DIRNAME / CONFIG_FILENAME
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def config_path() -> Path:
|
|
113
|
+
"""The config file to READ, new location first.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
`<xdg base>/hyperun/config.json` when it exists; otherwise the old
|
|
117
|
+
`ddpsrun` path when THAT exists; otherwise the new path, so a
|
|
118
|
+
"not logged in" message names the place a login will write.
|
|
119
|
+
"""
|
|
120
|
+
current = config_dir() / CONFIG_FILENAME
|
|
121
|
+
if current.exists():
|
|
122
|
+
return current
|
|
123
|
+
legacy = legacy_config_path()
|
|
124
|
+
return legacy if legacy.exists() else current
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def save(credentials: Credentials) -> Path:
|
|
128
|
+
"""Write the credentials, readable by this user only.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
credentials: what `hyperun login` collected.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
The path written, so the caller can tell the user where it went.
|
|
135
|
+
|
|
136
|
+
The chmod happens AFTER the write and the file is opened with 0600 from the
|
|
137
|
+
start: creating it world-readable and narrowing it afterwards would leave a
|
|
138
|
+
window in which another user on a shared machine could read the token.
|
|
139
|
+
"""
|
|
140
|
+
directory = config_dir()
|
|
141
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
142
|
+
os.chmod(directory, stat.S_IRWXU)
|
|
143
|
+
|
|
144
|
+
path = config_path()
|
|
145
|
+
# O_CREAT | O_WRONLY | O_TRUNC with mode 0600, rather than open(path, "w"),
|
|
146
|
+
# which would create it with the process umask applied to 0666.
|
|
147
|
+
descriptor = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, stat.S_IRUSR | stat.S_IWUSR)
|
|
148
|
+
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
149
|
+
document = {"server": credentials.server, "token": credentials.token}
|
|
150
|
+
# Only written when there is one. A file from a static-token login stays
|
|
151
|
+
# exactly the shape it was before Cognito existed.
|
|
152
|
+
if credentials.refresh_token:
|
|
153
|
+
document["refresh_token"] = credentials.refresh_token
|
|
154
|
+
json.dump(document, handle, indent=2)
|
|
155
|
+
handle.write("\n")
|
|
156
|
+
return path
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def load() -> Credentials:
|
|
160
|
+
"""Find the credentials to use.
|
|
161
|
+
|
|
162
|
+
Order: environment first, then the file. The environment wins so that a
|
|
163
|
+
script or a CI job can override a developer's own login without touching
|
|
164
|
+
their file.
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
The credentials.
|
|
168
|
+
|
|
169
|
+
Raises:
|
|
170
|
+
NotLoggedIn: neither source had both values. The message names the
|
|
171
|
+
command that fixes it, because "not logged in" on its own has sent
|
|
172
|
+
more than one person to the documentation.
|
|
173
|
+
"""
|
|
174
|
+
# New spelling first, old one second. Both are read so a shell profile or CI
|
|
175
|
+
# file that exports the pre-rename names keeps working.
|
|
176
|
+
server = (os.environ.get(SERVER_ENV, "")
|
|
177
|
+
or os.environ.get(LEGACY_SERVER_ENV, "")).strip()
|
|
178
|
+
token = (os.environ.get(TOKEN_ENV, "")
|
|
179
|
+
or os.environ.get(LEGACY_TOKEN_ENV, "")).strip()
|
|
180
|
+
if server and token:
|
|
181
|
+
return Credentials(server=server.rstrip("/"), token=token)
|
|
182
|
+
|
|
183
|
+
path = config_path()
|
|
184
|
+
if not path.exists():
|
|
185
|
+
raise NotLoggedIn(
|
|
186
|
+
f"not logged in. Run:\n"
|
|
187
|
+
f" hyperun login --server <url>\n"
|
|
188
|
+
f"or set {SERVER_ENV} and {TOKEN_ENV}."
|
|
189
|
+
)
|
|
190
|
+
try:
|
|
191
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
192
|
+
document = json.load(handle)
|
|
193
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
194
|
+
raise NotLoggedIn(f"{path} is unreadable ({exc}). Run `hyperun login` again.") from exc
|
|
195
|
+
|
|
196
|
+
file_server = str(document.get("server", "")).strip()
|
|
197
|
+
file_token = str(document.get("token", "")).strip()
|
|
198
|
+
if not file_server or not file_token:
|
|
199
|
+
raise NotLoggedIn(f"{path} is missing a server or a token. Run `hyperun login` again.")
|
|
200
|
+
|
|
201
|
+
return Credentials(
|
|
202
|
+
server=(server or file_server).rstrip("/"),
|
|
203
|
+
token=token or file_token,
|
|
204
|
+
refresh_token=str(document.get("refresh_token", "")).strip(),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def forget() -> bool:
|
|
209
|
+
"""Delete the stored credentials.
|
|
210
|
+
|
|
211
|
+
Returns:
|
|
212
|
+
True if a file was removed, False if there was nothing to remove.
|
|
213
|
+
"""
|
|
214
|
+
path = config_path()
|
|
215
|
+
if not path.exists():
|
|
216
|
+
return False
|
|
217
|
+
path.unlink()
|
|
218
|
+
return True
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hyperun
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Submit a GPU job and get results back. No kubectl, no cloud account.
|
|
5
|
+
Author: DDPS Lab
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/ddps-lab/pacsrun-gw
|
|
8
|
+
Project-URL: Repository, https://github.com/ddps-lab/pacsrun-gw
|
|
9
|
+
Project-URL: Issues, https://github.com/ddps-lab/pacsrun-gw/issues
|
|
10
|
+
Keywords: gpu,kubernetes,batch,jobs,machine-learning,cloud
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: requests>=2.31
|
|
30
|
+
Requires-Dist: PyYAML>=6.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
33
|
+
Requires-Dist: responses>=0.25; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# hyperun
|
|
37
|
+
|
|
38
|
+
**Submit a GPU job. Get results back. No `kubectl`, no cloud account, no VM to babysit.**
|
|
39
|
+
|
|
40
|
+
`hyperun` is the front door to PACSrun, a Kubernetes-native batch job system that rents GPUs from
|
|
41
|
+
whichever vendor is cheapest for the shape you asked for — AWS, GCP or RunPod — runs your code
|
|
42
|
+
there, and puts the results in object storage. You describe the job; you never see the machine.
|
|
43
|
+
|
|
44
|
+
```console
|
|
45
|
+
$ hyperun login <token>
|
|
46
|
+
$ hyperun estimate -f job.yaml
|
|
47
|
+
$ hyperun submit -f job.yaml
|
|
48
|
+
$ hyperun logs <job-id> --follow
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Why it exists
|
|
52
|
+
|
|
53
|
+
A researcher who wants a GPU for two hours has three bad options today: get a cloud account and
|
|
54
|
+
learn its console, get `kubectl` access to somebody's cluster, or ask a person. Each of those
|
|
55
|
+
teaches infrastructure to somebody whose job is not infrastructure.
|
|
56
|
+
|
|
57
|
+
The alternative here is that **a batch job is the unit**. You hand over a job description and a
|
|
58
|
+
program. The platform decides which machine to rent, rents it, runs your code, brings the output
|
|
59
|
+
home, and hands the machine back. When it goes wrong you get an exit code and a sentence, not a
|
|
60
|
+
half-configured VM.
|
|
61
|
+
|
|
62
|
+
## Install
|
|
63
|
+
|
|
64
|
+
```console
|
|
65
|
+
pip install hyperun
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Two dependencies, `requests` and `PyYAML`, and that is deliberate: this package installs next to
|
|
69
|
+
your research code, so every extra dependency is one that can fight with something you have
|
|
70
|
+
pinned. No `click`, no `rich`, no `pydantic`.
|
|
71
|
+
|
|
72
|
+
Python 3.9 or newer.
|
|
73
|
+
|
|
74
|
+
## Configure
|
|
75
|
+
|
|
76
|
+
`hyperun` needs a server URL and a token. Ask whoever runs your deployment for both.
|
|
77
|
+
|
|
78
|
+
```console
|
|
79
|
+
$ export HYPERUN_SERVER=https://<your-gateway-url>
|
|
80
|
+
$ hyperun login <token>
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`login` writes them to `~/.config/hyperun/config.json` with mode 0600, or to
|
|
84
|
+
`$XDG_CONFIG_HOME/hyperun` when that is set. A config left behind by the old `ddpsrun`
|
|
85
|
+
command is still read, so a rename logs nobody out. For CI, where a file is the wrong shape, set
|
|
86
|
+
`HYPERUN_TOKEN` in the environment and skip `login` entirely.
|
|
87
|
+
|
|
88
|
+
## The commands
|
|
89
|
+
|
|
90
|
+
| | |
|
|
91
|
+
|---|---|
|
|
92
|
+
| `login` / `logout` | store and remove the server URL and token |
|
|
93
|
+
| `explain` | what this deployment can do, in prose — run it first |
|
|
94
|
+
| `schema` | the job description's fields and their types |
|
|
95
|
+
| `estimate` | what a job would cost and how long it would take, before you spend anything |
|
|
96
|
+
| `validate` | check a job description without submitting it |
|
|
97
|
+
| `submit` | submit it |
|
|
98
|
+
| `status` / `watch` | where a job is now; `watch` follows until it ends |
|
|
99
|
+
| `logs` | your program's own output, `--follow` to stream |
|
|
100
|
+
| `stats` | GPU utilisation, memory, temperature and power while the job runs |
|
|
101
|
+
| `cancel` | stop a job and hand its machine back |
|
|
102
|
+
|
|
103
|
+
Exit codes are contractual: **0** it worked, **1** the server refused or a `validate` finding would
|
|
104
|
+
stop the job, **2** the command was wrong or there are no credentials.
|
|
105
|
+
|
|
106
|
+
## Writing a job
|
|
107
|
+
|
|
108
|
+
A job description is YAML:
|
|
109
|
+
|
|
110
|
+
```yaml
|
|
111
|
+
image: nvcr.io/nvidia/pytorch:24.10-py3
|
|
112
|
+
command: ["/bin/bash", "-c"]
|
|
113
|
+
args:
|
|
114
|
+
- |
|
|
115
|
+
python train.py --epochs 3
|
|
116
|
+
resources:
|
|
117
|
+
gpus:
|
|
118
|
+
name: L4
|
|
119
|
+
count: 1
|
|
120
|
+
resultPath: s3://<your-bucket>/runs/my-experiment/
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Run `hyperun schema` for every field and `hyperun explain` for what your deployment allows.
|
|
124
|
+
|
|
125
|
+
**Do not guess the GPU or the runtime.** `hyperun estimate` answers both from measured data, and
|
|
126
|
+
guessing is how a job ends up on a card several times more expensive than it needed.
|
|
127
|
+
|
|
128
|
+
## Using it from an AI coding agent
|
|
129
|
+
|
|
130
|
+
This repository ships a [Claude Code](https://claude.com/claude-code) plugin. It teaches an agent
|
|
131
|
+
five steps: read what the deployment allows, read your repository against the script contract,
|
|
132
|
+
estimate rather than assume, validate and stop on a finding, and submit only after you approve.
|
|
133
|
+
|
|
134
|
+
```
|
|
135
|
+
/plugin marketplace add ddps-lab/pacsrun-gw
|
|
136
|
+
/plugin install hyperun
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The plugin's reference documents are in [`agent/references/`](agent/references/). The one worth
|
|
140
|
+
reading yourself is [`script-contract.md`](agent/references/script-contract.md): a list of the ways
|
|
141
|
+
a long training run dies in its last minute, each one taken from a run that did.
|
|
142
|
+
|
|
143
|
+
## What this is not
|
|
144
|
+
|
|
145
|
+
- **Not an interactive machine.** There is no SSH and no shell into a running job. You submit a
|
|
146
|
+
program and read its output. If you need to poke at a live container, this is the wrong tool
|
|
147
|
+
today.
|
|
148
|
+
- **Not a scheduler you host.** `hyperun` is a client. Somebody has to run the gateway and the
|
|
149
|
+
PACSrun operator; see [`docs/`](docs/) if that somebody is you.
|
|
150
|
+
- **Not free of limits.** A job's credentials for writing results last twelve hours, which is
|
|
151
|
+
AWS's hard maximum for a role session and not a setting anyone can raise. A run longer than that
|
|
152
|
+
must checkpoint and resume, and the driver warns half an hour before the deadline.
|
|
153
|
+
|
|
154
|
+
## Repository layout
|
|
155
|
+
|
|
156
|
+
| | |
|
|
157
|
+
|---|---|
|
|
158
|
+
| `cli/` | the `hyperun` package published to PyPI |
|
|
159
|
+
| `server/` | the gateway: authenticates users, talks to Kubernetes |
|
|
160
|
+
| `agent/` | the Claude Code plugin — one skill, four reference documents |
|
|
161
|
+
| `ui/` | a small static web front end (`index.html`, `app.js`, `style.css`) |
|
|
162
|
+
| `terraform/` | the deployment: Lambda, Cognito, S3, CloudFront |
|
|
163
|
+
| `docs/` | seventeen design documents, numbered in reading order |
|
|
164
|
+
|
|
165
|
+
## Development
|
|
166
|
+
|
|
167
|
+
```console
|
|
168
|
+
$ pip install -e 'cli/[dev]' # the client
|
|
169
|
+
$ pip install -e 'server/[dev]' # the gateway
|
|
170
|
+
$ pytest cli/tests server/tests
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Both halves live in one repository so the two sides of an API change land in one commit. They share
|
|
174
|
+
nothing else: the client never imports the Kubernetes libraries, and the server never imports the
|
|
175
|
+
browser login flow.
|
|
176
|
+
|
|
177
|
+
**No account identifiers in this repository.** Documents and code use placeholders —
|
|
178
|
+
`<ACCOUNT_ID>`, `<RESULT_BUCKET>` — and CI checks every push for AWS account numbers, access keys,
|
|
179
|
+
GitHub and RunPod tokens, private keys, and Google OAuth secrets.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
Apache-2.0. See [`LICENSE`](LICENSE).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
hyperun/__init__.py,sha256=jqE7O6dnVn8OKFCTmtmvpCTHjGKkyTm7YprruIL1zFI,1213
|
|
2
|
+
hyperun/browser_login.py,sha256=gfiI2ZMtuJ8nyOVqJl0ILFKayAPsqDeEGILrePOa3tE,10219
|
|
3
|
+
hyperun/cli.py,sha256=nYYX1yTuRUABa3iikQL-EHKJEDQjAi9khJnkU9bYIjc,52853
|
|
4
|
+
hyperun/client.py,sha256=PIeS5PE2uKg8AiLN2N3G2gECWZKsMB_8BLK3ta9QlA0,10921
|
|
5
|
+
hyperun/config.py,sha256=mSxuY-_vP3yTejEiaf_poeAK0r170CT5MNmf5aeZTDM,8041
|
|
6
|
+
hyperun-0.2.0.dist-info/licenses/LICENSE,sha256=Gf64nZEw53YNYQFMUJSLB2ut7_ED3QKccibCx5T7jEo,11339
|
|
7
|
+
hyperun-0.2.0.dist-info/METADATA,sha256=3BhDu36T8oB0bfN2PZHXI2P-qe-vJA9-_OwmLr56OM4,7247
|
|
8
|
+
hyperun-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
9
|
+
hyperun-0.2.0.dist-info/entry_points.txt,sha256=WdF15yGaZtvGRDMp8r_pduSoyHQF5VsSlZRSeG7R5M4,45
|
|
10
|
+
hyperun-0.2.0.dist-info/top_level.txt,sha256=zc7oVfTtWl4PCmCogtnUwP_X3KSf7qLplW5yBeO3-B0,8
|
|
11
|
+
hyperun-0.2.0.dist-info/RECORD,,
|