runyard 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runyard/__init__.py +63 -0
- runyard/cli.py +219 -0
- runyard/client.py +73 -0
- runyard/core.py +485 -0
- runyard/integrations/__init__.py +2 -0
- runyard/integrations/accelerate.py +40 -0
- runyard/integrations/hf.py +51 -0
- runyard/integrations/keras.py +37 -0
- runyard/integrations/lightgbm.py +22 -0
- runyard/integrations/lightning.py +52 -0
- runyard/integrations/sklearn.py +61 -0
- runyard/integrations/torchtune.py +33 -0
- runyard/integrations/xgboost.py +45 -0
- runyard/media.py +95 -0
- runyard/wandb_hook.py +55 -0
- runyard/wandb_shim.py +256 -0
- runyard/watch.py +102 -0
- runyard-0.3.0.dist-info/METADATA +50 -0
- runyard-0.3.0.dist-info/RECORD +22 -0
- runyard-0.3.0.dist-info/WHEEL +4 -0
- runyard-0.3.0.dist-info/entry_points.txt +2 -0
- runyard_wandb.pth +1 -0
runyard/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""runyard: a lightweight, self-hosted training tracker.
|
|
2
|
+
|
|
3
|
+
import runyard as ry
|
|
4
|
+
run = ry.init(project="gba-diffusion", name="pixel32_h512", config={"size": 32})
|
|
5
|
+
run.log({"loss": 0.031}, step=1000)
|
|
6
|
+
run.image("samples", grid, step=1000)
|
|
7
|
+
run.finish()
|
|
8
|
+
|
|
9
|
+
Settings come from RUNYARD_SERVER, RUNYARD_API_KEY and RUNYARD_PROJECT.
|
|
10
|
+
"""
|
|
11
|
+
from .core import Run, sync, warn
|
|
12
|
+
|
|
13
|
+
__version__ = "0.3.0"
|
|
14
|
+
|
|
15
|
+
run = None # the run the module-level shortcuts act on: the last one init() made
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def init(project=None, name=None, config=None, **kwargs):
|
|
19
|
+
"""Starts a run. kwargs: tags, notes, id, server, api_key, capture_output, system."""
|
|
20
|
+
global run
|
|
21
|
+
run = Run(project, name, config, **kwargs)
|
|
22
|
+
return run
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _current(name):
|
|
26
|
+
if run is None:
|
|
27
|
+
warn(f"{name}() called before init(), ignored", once=True)
|
|
28
|
+
return run
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def log(data, step=None):
|
|
32
|
+
if _current("log"):
|
|
33
|
+
run.log(data, step=step)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def image(key, images, step=None, caption=None):
|
|
37
|
+
if _current("image"):
|
|
38
|
+
run.image(key, images, step=step, caption=caption)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def video(key, video, fps=30, step=None, caption=None):
|
|
42
|
+
if _current("video"):
|
|
43
|
+
run.video(key, video, fps=fps, step=step, caption=caption)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def text(key, rows, step=None):
|
|
47
|
+
if _current("text"):
|
|
48
|
+
run.text(key, rows, step=step)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def artifact(path, name=None, kind="file", step=None, upload=False, meta=None):
|
|
52
|
+
if _current("artifact"):
|
|
53
|
+
run.artifact(path, name=name, kind=kind, step=step, upload=upload, meta=meta)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def alert(title, body="", level="info"):
|
|
57
|
+
if _current("alert"):
|
|
58
|
+
run.alert(title, body=body, level=level)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def finish(status="finished"):
|
|
62
|
+
if _current("finish"):
|
|
63
|
+
run.finish(status)
|
runyard/cli.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""runyard command line. Every read command takes --json for scripts and agents.
|
|
2
|
+
|
|
3
|
+
runyard runs ls [--project P] [--status S]
|
|
4
|
+
runyard runs show RUN RUN: id, id prefix or run name
|
|
5
|
+
runyard logs RUN [-n 100] [-f]
|
|
6
|
+
runyard samples RUN [--out DIR] downloads the latest samples
|
|
7
|
+
runyard artifacts RUN [--get NAME] checkpoints and files; --get downloads an uploaded one
|
|
8
|
+
runyard sync sends runs saved while the server was unreachable
|
|
9
|
+
runyard watch DIR [--project P] [--once] [--interval 10]
|
|
10
|
+
sends TensorBoard event files under DIR, one run per directory, until Ctrl-C;
|
|
11
|
+
--once reads them once and exits (needs the tensorboard package)
|
|
12
|
+
"""
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import signal
|
|
17
|
+
import sys
|
|
18
|
+
import time
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import urllib.request
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from . import __version__
|
|
25
|
+
from .client import HTTPError, request
|
|
26
|
+
from .core import sync
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def server():
|
|
30
|
+
return (os.environ.get("RUNYARD_SERVER") or "http://localhost:8080").rstrip("/")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def key():
|
|
34
|
+
return os.environ.get("RUNYARD_API_KEY", "") # reads need it only when the server runs with RUNYARD_AUTH
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def api(path):
|
|
38
|
+
return request("GET", server() + "/api/v1" + path, key())
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def download(path, dst):
|
|
42
|
+
req = urllib.request.Request(server() + path, headers={"Authorization": "Bearer " + key()} if key() else {})
|
|
43
|
+
with urllib.request.urlopen(req, timeout=300) as r, open(dst, "wb") as f:
|
|
44
|
+
while chunk := r.read(1 << 20):
|
|
45
|
+
f.write(chunk)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def dump(v):
|
|
49
|
+
print(json.dumps(v, indent=1))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def ago(ts):
|
|
53
|
+
s = (datetime.now(timezone.utc) - datetime.fromisoformat(ts)).total_seconds()
|
|
54
|
+
for unit, n in (("d", 86400), ("h", 3600), ("m", 60)):
|
|
55
|
+
if s >= n:
|
|
56
|
+
return f"{int(s // n)}{unit} ago"
|
|
57
|
+
return f"{int(s)}s ago"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def cmd_runs_ls(a):
|
|
61
|
+
q = urllib.parse.urlencode({k: v for k, v in (("project", a.project), ("status", a.status), ("limit", a.limit)) if v})
|
|
62
|
+
runs = api("/runs?" + q)
|
|
63
|
+
if a.json:
|
|
64
|
+
return dump(runs)
|
|
65
|
+
for r in runs:
|
|
66
|
+
print(f"{r['id'][:8]} {r['status']:<8} {r['project']}/{r['name']} started {ago(r['started_at'])}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_runs_show(a):
|
|
70
|
+
r = api("/runs/" + urllib.parse.quote(a.run))
|
|
71
|
+
if a.json:
|
|
72
|
+
return dump(r)
|
|
73
|
+
print(f"{r['project']}/{r['name']} {r['status']} id {r['id']}")
|
|
74
|
+
print(f"started {ago(r['started_at'])} on {r['host'] or '?'}" + (f", ended {ago(r['ended_at'])}" if r["ended_at"] else ""))
|
|
75
|
+
for title, d in (("config", r["config"]), ("summary", r["summary"])):
|
|
76
|
+
if d:
|
|
77
|
+
print(f"\n{title}:")
|
|
78
|
+
for k, v in d.items():
|
|
79
|
+
print(f" {k}: {v}")
|
|
80
|
+
if r.get("metrics"):
|
|
81
|
+
print("\nmetrics (last / min / max, last step):")
|
|
82
|
+
for k, m in sorted(r["metrics"].items()):
|
|
83
|
+
print(f" {k}: {m['last']} / {m['min']} / {m['max']}, step {m['step']}")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def cmd_logs(a):
|
|
87
|
+
r = api("/runs/" + urllib.parse.quote(a.run))
|
|
88
|
+
lines = api(f"/runs/{r['id']}/logs?tail&limit={a.n}")
|
|
89
|
+
after = 0
|
|
90
|
+
while True:
|
|
91
|
+
for line in lines:
|
|
92
|
+
print(json.dumps(line) if a.json else line["line"], flush=True)
|
|
93
|
+
if not a.follow:
|
|
94
|
+
return
|
|
95
|
+
if not lines and api("/runs/" + r["id"])["status"] != "running":
|
|
96
|
+
return
|
|
97
|
+
after = lines[-1]["seq"] if lines else after
|
|
98
|
+
time.sleep(2)
|
|
99
|
+
lines = api(f"/runs/{r['id']}/logs?after={after}&limit=1000")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def cmd_samples(a):
|
|
103
|
+
r = api("/runs/" + urllib.parse.quote(a.run))
|
|
104
|
+
media = api(f"/runs/{r['id']}/media" + (f"?key={urllib.parse.quote(a.key)}" if a.key else ""))
|
|
105
|
+
latest = {}
|
|
106
|
+
for m in media:
|
|
107
|
+
latest[m["key"]] = max(latest.get(m["key"], m["step"]), m["step"])
|
|
108
|
+
media = [m for m in media if m["step"] == latest[m["key"]]]
|
|
109
|
+
out = Path(a.out or f"runyard-samples/{r['name']}")
|
|
110
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
for m in media:
|
|
112
|
+
dst = out / f"{m['key'].replace('/', '_')}_{m['step']}_{m['id'][:8]}{Path(m['url']).suffix}"
|
|
113
|
+
download(m["url"], dst)
|
|
114
|
+
m["file"] = str(dst)
|
|
115
|
+
if not a.json:
|
|
116
|
+
print(dst)
|
|
117
|
+
if a.json:
|
|
118
|
+
dump(media)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def cmd_artifacts(a):
|
|
122
|
+
r = api("/runs/" + urllib.parse.quote(a.run))
|
|
123
|
+
arts = api(f"/runs/{r['id']}/artifacts")
|
|
124
|
+
if a.get:
|
|
125
|
+
arts = [x for x in arts if a.get in (x["id"], x["name"]) and x.get("url")]
|
|
126
|
+
if not arts:
|
|
127
|
+
print(f"runyard: no uploaded artifact {a.get!r} in {r['name']}", file=sys.stderr)
|
|
128
|
+
return 1
|
|
129
|
+
out = Path(a.out or ".")
|
|
130
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
dst = out / Path(arts[0]["name"]).name
|
|
132
|
+
download(arts[0]["url"], dst)
|
|
133
|
+
return print(dst)
|
|
134
|
+
if a.json:
|
|
135
|
+
return dump(arts)
|
|
136
|
+
for x in arts:
|
|
137
|
+
size = f"{x['size'] / 1e6:.1f} MB" if x.get("size") is not None else "?"
|
|
138
|
+
where = "uploaded" if x.get("url") else x["uri"]
|
|
139
|
+
print(f"{x['kind']:<10} {x['name']} step {x['step']} {size} {where}")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def cmd_sync(a):
|
|
143
|
+
"""Sends what runs saved at exit while their server was unreachable."""
|
|
144
|
+
from . import core
|
|
145
|
+
|
|
146
|
+
files = sorted((core.CACHE / "pending").glob("*.json"))
|
|
147
|
+
left = [f for f in files if not sync(f)]
|
|
148
|
+
print(f"sent {len(files) - len(left)} of {len(files)} pending runs")
|
|
149
|
+
return 1 if left else 0
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def cmd_watch(a):
|
|
153
|
+
"""Sends TensorBoard event files under DIR, one run per directory; polls for new events until Ctrl-C."""
|
|
154
|
+
if not Path(a.dir).is_dir():
|
|
155
|
+
print(f"runyard: {a.dir} is not a directory", file=sys.stderr)
|
|
156
|
+
return 1
|
|
157
|
+
try:
|
|
158
|
+
from .watch import watch
|
|
159
|
+
except ImportError as e:
|
|
160
|
+
print(f"runyard: watch needs tensorboard, pip install 'runyard[tensorboard]' ({e})", file=sys.stderr)
|
|
161
|
+
return 1
|
|
162
|
+
if not a.once:
|
|
163
|
+
signal.signal(signal.SIGTERM, signal.default_int_handler) # stop like Ctrl-C, so the runs end finished
|
|
164
|
+
watch(a.dir, a.project, a.once, a.interval)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def main(argv=None):
|
|
168
|
+
p = argparse.ArgumentParser(prog="runyard", description="Reads runs from the server in RUNYARD_SERVER; watch sends to it.")
|
|
169
|
+
p.add_argument("--version", action="version", version=__version__)
|
|
170
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
171
|
+
js = argparse.ArgumentParser(add_help=False)
|
|
172
|
+
js.add_argument("--json", action="store_true", help="print JSON")
|
|
173
|
+
|
|
174
|
+
runs = sub.add_parser("runs", help="list or show runs").add_subparsers(dest="sub", required=True)
|
|
175
|
+
ls = runs.add_parser("ls", parents=[js], help="list runs, newest first")
|
|
176
|
+
ls.add_argument("--project")
|
|
177
|
+
ls.add_argument("--status", choices=["running", "finished", "failed", "crashed"])
|
|
178
|
+
ls.add_argument("--limit", type=int, default=30)
|
|
179
|
+
ls.set_defaults(fn=cmd_runs_ls)
|
|
180
|
+
show = runs.add_parser("show", parents=[js], help="config, summary and metrics of a run")
|
|
181
|
+
show.add_argument("run", help="id, id prefix or run name")
|
|
182
|
+
show.set_defaults(fn=cmd_runs_show)
|
|
183
|
+
|
|
184
|
+
logs = sub.add_parser("logs", parents=[js], help="a run's output")
|
|
185
|
+
logs.add_argument("run")
|
|
186
|
+
logs.add_argument("-n", type=int, default=100, help="last N lines (default 100)")
|
|
187
|
+
logs.add_argument("-f", "--follow", action="store_true", help="keep printing new lines while the run is running")
|
|
188
|
+
logs.set_defaults(fn=cmd_logs)
|
|
189
|
+
|
|
190
|
+
samples = sub.add_parser("samples", parents=[js], help="download a run's latest samples")
|
|
191
|
+
samples.add_argument("run")
|
|
192
|
+
samples.add_argument("--key")
|
|
193
|
+
samples.add_argument("--out", help="directory (default runyard-samples/<run>)")
|
|
194
|
+
samples.set_defaults(fn=cmd_samples)
|
|
195
|
+
|
|
196
|
+
arts = sub.add_parser("artifacts", parents=[js], help="a run's checkpoints and files; --get downloads an uploaded one")
|
|
197
|
+
arts.add_argument("run")
|
|
198
|
+
arts.add_argument("--get", metavar="NAME", help="download the uploaded artifact with this name or id")
|
|
199
|
+
arts.add_argument("--out", help="directory for --get (default: here)")
|
|
200
|
+
arts.set_defaults(fn=cmd_artifacts)
|
|
201
|
+
|
|
202
|
+
sub.add_parser("sync", help=cmd_sync.__doc__).set_defaults(fn=cmd_sync)
|
|
203
|
+
|
|
204
|
+
watch = sub.add_parser("watch", help=cmd_watch.__doc__)
|
|
205
|
+
watch.add_argument("dir")
|
|
206
|
+
watch.add_argument("--project", help="default: RUNYARD_PROJECT, else the directory's name")
|
|
207
|
+
watch.add_argument("--once", action="store_true", help="read everything once, finish the runs and exit (imports)")
|
|
208
|
+
watch.add_argument("--interval", type=float, default=10, help="seconds between polls (default 10)")
|
|
209
|
+
watch.set_defaults(fn=cmd_watch)
|
|
210
|
+
a = p.parse_args(argv)
|
|
211
|
+
try:
|
|
212
|
+
return a.fn(a) or 0
|
|
213
|
+
except (OSError, HTTPError) as e: # unreachable server, or it refused the request
|
|
214
|
+
print(f"runyard: {e}", file=sys.stderr)
|
|
215
|
+
return 1
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
if __name__ == "__main__":
|
|
219
|
+
sys.exit(main())
|
runyard/client.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Minimal HTTP client for the runyard REST API (standard library only)."""
|
|
2
|
+
import json
|
|
3
|
+
import math
|
|
4
|
+
import urllib.error
|
|
5
|
+
import urllib.request
|
|
6
|
+
import uuid
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class HTTPError(Exception):
|
|
11
|
+
def __init__(self, status, body):
|
|
12
|
+
super().__init__(f"HTTP {status}: {body[:300]}")
|
|
13
|
+
self.status, self.body = status, body
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def clean(v):
|
|
17
|
+
"""JSON-safe copy: NaN/±Inf become strings (JSON has no such numbers), tuples lists, unknown objects str."""
|
|
18
|
+
if isinstance(v, float) and not math.isfinite(v):
|
|
19
|
+
return "NaN" if math.isnan(v) else ("Infinity" if v > 0 else "-Infinity")
|
|
20
|
+
if isinstance(v, dict):
|
|
21
|
+
return {str(k): clean(x) for k, x in v.items()}
|
|
22
|
+
if isinstance(v, (list, tuple)):
|
|
23
|
+
return [clean(x) for x in v]
|
|
24
|
+
if v is None or isinstance(v, (str, int, float, bool)):
|
|
25
|
+
return v
|
|
26
|
+
if hasattr(v, "item"): # numpy / torch scalar
|
|
27
|
+
try:
|
|
28
|
+
return clean(v.item())
|
|
29
|
+
except Exception:
|
|
30
|
+
pass
|
|
31
|
+
return str(v)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _stream(head, path, tail):
|
|
35
|
+
yield head
|
|
36
|
+
with open(path, "rb") as f:
|
|
37
|
+
while True:
|
|
38
|
+
chunk = f.read(1 << 20)
|
|
39
|
+
if not chunk:
|
|
40
|
+
break
|
|
41
|
+
yield chunk
|
|
42
|
+
yield tail
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def request(method, url, key="", body=None, file=None, timeout=10):
|
|
46
|
+
"""Sends body as JSON, or as multipart fields when file=(filename, bytes or Path) is given; a Path is streamed from
|
|
47
|
+
disk, so a large checkpoint never sits in memory. Returns the decoded reply."""
|
|
48
|
+
headers = {"Authorization": "Bearer " + key} if key else {}
|
|
49
|
+
data = None
|
|
50
|
+
if file is not None:
|
|
51
|
+
boundary = uuid.uuid4().hex
|
|
52
|
+
parts = []
|
|
53
|
+
for k, v in (body or {}).items():
|
|
54
|
+
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode())
|
|
55
|
+
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{file[0]}"\r\n'
|
|
56
|
+
f"Content-Type: application/octet-stream\r\n\r\n".encode())
|
|
57
|
+
head, tail = b"".join(parts), f"\r\n--{boundary}--\r\n".encode()
|
|
58
|
+
if isinstance(file[1], Path):
|
|
59
|
+
data = _stream(head, file[1], tail)
|
|
60
|
+
headers["Content-Length"] = str(len(head) + file[1].stat().st_size + len(tail))
|
|
61
|
+
else:
|
|
62
|
+
data = head + file[1] + tail
|
|
63
|
+
headers["Content-Type"] = "multipart/form-data; boundary=" + boundary
|
|
64
|
+
elif body is not None:
|
|
65
|
+
data = json.dumps(clean(body), allow_nan=False).encode()
|
|
66
|
+
headers["Content-Type"] = "application/json"
|
|
67
|
+
req = urllib.request.Request(url, data=data, method=method, headers=headers)
|
|
68
|
+
try:
|
|
69
|
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
70
|
+
raw = r.read()
|
|
71
|
+
except urllib.error.HTTPError as e:
|
|
72
|
+
raise HTTPError(e.code, e.read().decode(errors="replace")) from None
|
|
73
|
+
return json.loads(raw) if raw else None
|