runyard 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
runyard/__init__.py ADDED
@@ -0,0 +1,63 @@
1
+ """runyard: a lightweight, self-hosted training tracker.
2
+
3
+ import runyard as ry
4
+ run = ry.init(project="gba-diffusion", name="pixel32_h512", config={"size": 32})
5
+ run.log({"loss": 0.031}, step=1000)
6
+ run.image("samples", grid, step=1000)
7
+ run.finish()
8
+
9
+ Settings come from RUNYARD_SERVER, RUNYARD_API_KEY and RUNYARD_PROJECT.
10
+ """
11
+ from .core import Run, sync, warn
12
+
13
+ __version__ = "0.3.0"
14
+
15
+ run = None # the run the module-level shortcuts act on: the last one init() made
16
+
17
+
18
+ def init(project=None, name=None, config=None, **kwargs):
19
+ """Starts a run. kwargs: tags, notes, id, server, api_key, capture_output, system."""
20
+ global run
21
+ run = Run(project, name, config, **kwargs)
22
+ return run
23
+
24
+
25
+ def _current(name):
26
+ if run is None:
27
+ warn(f"{name}() called before init(), ignored", once=True)
28
+ return run
29
+
30
+
31
+ def log(data, step=None):
32
+ if _current("log"):
33
+ run.log(data, step=step)
34
+
35
+
36
+ def image(key, images, step=None, caption=None):
37
+ if _current("image"):
38
+ run.image(key, images, step=step, caption=caption)
39
+
40
+
41
+ def video(key, video, fps=30, step=None, caption=None):
42
+ if _current("video"):
43
+ run.video(key, video, fps=fps, step=step, caption=caption)
44
+
45
+
46
+ def text(key, rows, step=None):
47
+ if _current("text"):
48
+ run.text(key, rows, step=step)
49
+
50
+
51
+ def artifact(path, name=None, kind="file", step=None, upload=False, meta=None):
52
+ if _current("artifact"):
53
+ run.artifact(path, name=name, kind=kind, step=step, upload=upload, meta=meta)
54
+
55
+
56
+ def alert(title, body="", level="info"):
57
+ if _current("alert"):
58
+ run.alert(title, body=body, level=level)
59
+
60
+
61
+ def finish(status="finished"):
62
+ if _current("finish"):
63
+ run.finish(status)
runyard/cli.py ADDED
@@ -0,0 +1,219 @@
1
+ """runyard command line. Every read command takes --json for scripts and agents.
2
+
3
+ runyard runs ls [--project P] [--status S]
4
+ runyard runs show RUN RUN: id, id prefix or run name
5
+ runyard logs RUN [-n 100] [-f]
6
+ runyard samples RUN [--out DIR] downloads the latest samples
7
+ runyard artifacts RUN [--get NAME] checkpoints and files; --get downloads an uploaded one
8
+ runyard sync sends runs saved while the server was unreachable
9
+ runyard watch DIR [--project P] [--once] [--interval 10]
10
+ sends TensorBoard event files under DIR, one run per directory, until Ctrl-C;
11
+ --once reads them once and exits (needs the tensorboard package)
12
+ """
13
+ import argparse
14
+ import json
15
+ import os
16
+ import signal
17
+ import sys
18
+ import time
19
+ import urllib.parse
20
+ import urllib.request
21
+ from datetime import datetime, timezone
22
+ from pathlib import Path
23
+
24
+ from . import __version__
25
+ from .client import HTTPError, request
26
+ from .core import sync
27
+
28
+
29
+ def server():
30
+ return (os.environ.get("RUNYARD_SERVER") or "http://localhost:8080").rstrip("/")
31
+
32
+
33
+ def key():
34
+ return os.environ.get("RUNYARD_API_KEY", "") # reads need it only when the server runs with RUNYARD_AUTH
35
+
36
+
37
+ def api(path):
38
+ return request("GET", server() + "/api/v1" + path, key())
39
+
40
+
41
+ def download(path, dst):
42
+ req = urllib.request.Request(server() + path, headers={"Authorization": "Bearer " + key()} if key() else {})
43
+ with urllib.request.urlopen(req, timeout=300) as r, open(dst, "wb") as f:
44
+ while chunk := r.read(1 << 20):
45
+ f.write(chunk)
46
+
47
+
48
+ def dump(v):
49
+ print(json.dumps(v, indent=1))
50
+
51
+
52
+ def ago(ts):
53
+ s = (datetime.now(timezone.utc) - datetime.fromisoformat(ts)).total_seconds()
54
+ for unit, n in (("d", 86400), ("h", 3600), ("m", 60)):
55
+ if s >= n:
56
+ return f"{int(s // n)}{unit} ago"
57
+ return f"{int(s)}s ago"
58
+
59
+
60
+ def cmd_runs_ls(a):
61
+ q = urllib.parse.urlencode({k: v for k, v in (("project", a.project), ("status", a.status), ("limit", a.limit)) if v})
62
+ runs = api("/runs?" + q)
63
+ if a.json:
64
+ return dump(runs)
65
+ for r in runs:
66
+ print(f"{r['id'][:8]} {r['status']:<8} {r['project']}/{r['name']} started {ago(r['started_at'])}")
67
+
68
+
69
+ def cmd_runs_show(a):
70
+ r = api("/runs/" + urllib.parse.quote(a.run))
71
+ if a.json:
72
+ return dump(r)
73
+ print(f"{r['project']}/{r['name']} {r['status']} id {r['id']}")
74
+ print(f"started {ago(r['started_at'])} on {r['host'] or '?'}" + (f", ended {ago(r['ended_at'])}" if r["ended_at"] else ""))
75
+ for title, d in (("config", r["config"]), ("summary", r["summary"])):
76
+ if d:
77
+ print(f"\n{title}:")
78
+ for k, v in d.items():
79
+ print(f" {k}: {v}")
80
+ if r.get("metrics"):
81
+ print("\nmetrics (last / min / max, last step):")
82
+ for k, m in sorted(r["metrics"].items()):
83
+ print(f" {k}: {m['last']} / {m['min']} / {m['max']}, step {m['step']}")
84
+
85
+
86
+ def cmd_logs(a):
87
+ r = api("/runs/" + urllib.parse.quote(a.run))
88
+ lines = api(f"/runs/{r['id']}/logs?tail&limit={a.n}")
89
+ after = 0
90
+ while True:
91
+ for line in lines:
92
+ print(json.dumps(line) if a.json else line["line"], flush=True)
93
+ if not a.follow:
94
+ return
95
+ if not lines and api("/runs/" + r["id"])["status"] != "running":
96
+ return
97
+ after = lines[-1]["seq"] if lines else after
98
+ time.sleep(2)
99
+ lines = api(f"/runs/{r['id']}/logs?after={after}&limit=1000")
100
+
101
+
102
+ def cmd_samples(a):
103
+ r = api("/runs/" + urllib.parse.quote(a.run))
104
+ media = api(f"/runs/{r['id']}/media" + (f"?key={urllib.parse.quote(a.key)}" if a.key else ""))
105
+ latest = {}
106
+ for m in media:
107
+ latest[m["key"]] = max(latest.get(m["key"], m["step"]), m["step"])
108
+ media = [m for m in media if m["step"] == latest[m["key"]]]
109
+ out = Path(a.out or f"runyard-samples/{r['name']}")
110
+ out.mkdir(parents=True, exist_ok=True)
111
+ for m in media:
112
+ dst = out / f"{m['key'].replace('/', '_')}_{m['step']}_{m['id'][:8]}{Path(m['url']).suffix}"
113
+ download(m["url"], dst)
114
+ m["file"] = str(dst)
115
+ if not a.json:
116
+ print(dst)
117
+ if a.json:
118
+ dump(media)
119
+
120
+
121
+ def cmd_artifacts(a):
122
+ r = api("/runs/" + urllib.parse.quote(a.run))
123
+ arts = api(f"/runs/{r['id']}/artifacts")
124
+ if a.get:
125
+ arts = [x for x in arts if a.get in (x["id"], x["name"]) and x.get("url")]
126
+ if not arts:
127
+ print(f"runyard: no uploaded artifact {a.get!r} in {r['name']}", file=sys.stderr)
128
+ return 1
129
+ out = Path(a.out or ".")
130
+ out.mkdir(parents=True, exist_ok=True)
131
+ dst = out / Path(arts[0]["name"]).name
132
+ download(arts[0]["url"], dst)
133
+ return print(dst)
134
+ if a.json:
135
+ return dump(arts)
136
+ for x in arts:
137
+ size = f"{x['size'] / 1e6:.1f} MB" if x.get("size") is not None else "?"
138
+ where = "uploaded" if x.get("url") else x["uri"]
139
+ print(f"{x['kind']:<10} {x['name']} step {x['step']} {size} {where}")
140
+
141
+
142
+ def cmd_sync(a):
143
+ """Sends what runs saved at exit while their server was unreachable."""
144
+ from . import core
145
+
146
+ files = sorted((core.CACHE / "pending").glob("*.json"))
147
+ left = [f for f in files if not sync(f)]
148
+ print(f"sent {len(files) - len(left)} of {len(files)} pending runs")
149
+ return 1 if left else 0
150
+
151
+
152
+ def cmd_watch(a):
153
+ """Sends TensorBoard event files under DIR, one run per directory; polls for new events until Ctrl-C."""
154
+ if not Path(a.dir).is_dir():
155
+ print(f"runyard: {a.dir} is not a directory", file=sys.stderr)
156
+ return 1
157
+ try:
158
+ from .watch import watch
159
+ except ImportError as e:
160
+ print(f"runyard: watch needs tensorboard, pip install 'runyard[tensorboard]' ({e})", file=sys.stderr)
161
+ return 1
162
+ if not a.once:
163
+ signal.signal(signal.SIGTERM, signal.default_int_handler) # stop like Ctrl-C, so the runs end finished
164
+ watch(a.dir, a.project, a.once, a.interval)
165
+
166
+
167
+ def main(argv=None):
168
+ p = argparse.ArgumentParser(prog="runyard", description="Reads runs from the server in RUNYARD_SERVER; watch sends to it.")
169
+ p.add_argument("--version", action="version", version=__version__)
170
+ sub = p.add_subparsers(dest="cmd", required=True)
171
+ js = argparse.ArgumentParser(add_help=False)
172
+ js.add_argument("--json", action="store_true", help="print JSON")
173
+
174
+ runs = sub.add_parser("runs", help="list or show runs").add_subparsers(dest="sub", required=True)
175
+ ls = runs.add_parser("ls", parents=[js], help="list runs, newest first")
176
+ ls.add_argument("--project")
177
+ ls.add_argument("--status", choices=["running", "finished", "failed", "crashed"])
178
+ ls.add_argument("--limit", type=int, default=30)
179
+ ls.set_defaults(fn=cmd_runs_ls)
180
+ show = runs.add_parser("show", parents=[js], help="config, summary and metrics of a run")
181
+ show.add_argument("run", help="id, id prefix or run name")
182
+ show.set_defaults(fn=cmd_runs_show)
183
+
184
+ logs = sub.add_parser("logs", parents=[js], help="a run's output")
185
+ logs.add_argument("run")
186
+ logs.add_argument("-n", type=int, default=100, help="last N lines (default 100)")
187
+ logs.add_argument("-f", "--follow", action="store_true", help="keep printing new lines while the run is running")
188
+ logs.set_defaults(fn=cmd_logs)
189
+
190
+ samples = sub.add_parser("samples", parents=[js], help="download a run's latest samples")
191
+ samples.add_argument("run")
192
+ samples.add_argument("--key")
193
+ samples.add_argument("--out", help="directory (default runyard-samples/<run>)")
194
+ samples.set_defaults(fn=cmd_samples)
195
+
196
+ arts = sub.add_parser("artifacts", parents=[js], help="a run's checkpoints and files; --get downloads an uploaded one")
197
+ arts.add_argument("run")
198
+ arts.add_argument("--get", metavar="NAME", help="download the uploaded artifact with this name or id")
199
+ arts.add_argument("--out", help="directory for --get (default: here)")
200
+ arts.set_defaults(fn=cmd_artifacts)
201
+
202
+ sub.add_parser("sync", help=cmd_sync.__doc__).set_defaults(fn=cmd_sync)
203
+
204
+ watch = sub.add_parser("watch", help=cmd_watch.__doc__)
205
+ watch.add_argument("dir")
206
+ watch.add_argument("--project", help="default: RUNYARD_PROJECT, else the directory's name")
207
+ watch.add_argument("--once", action="store_true", help="read everything once, finish the runs and exit (imports)")
208
+ watch.add_argument("--interval", type=float, default=10, help="seconds between polls (default 10)")
209
+ watch.set_defaults(fn=cmd_watch)
210
+ a = p.parse_args(argv)
211
+ try:
212
+ return a.fn(a) or 0
213
+ except (OSError, HTTPError) as e: # unreachable server, or it refused the request
214
+ print(f"runyard: {e}", file=sys.stderr)
215
+ return 1
216
+
217
+
218
+ if __name__ == "__main__":
219
+ sys.exit(main())
runyard/client.py ADDED
@@ -0,0 +1,73 @@
1
+ """Minimal HTTP client for the runyard REST API (standard library only)."""
2
+ import json
3
+ import math
4
+ import urllib.error
5
+ import urllib.request
6
+ import uuid
7
+ from pathlib import Path
8
+
9
+
10
+ class HTTPError(Exception):
11
+ def __init__(self, status, body):
12
+ super().__init__(f"HTTP {status}: {body[:300]}")
13
+ self.status, self.body = status, body
14
+
15
+
16
+ def clean(v):
17
+ """JSON-safe copy: NaN/±Inf become strings (JSON has no such numbers), tuples lists, unknown objects str."""
18
+ if isinstance(v, float) and not math.isfinite(v):
19
+ return "NaN" if math.isnan(v) else ("Infinity" if v > 0 else "-Infinity")
20
+ if isinstance(v, dict):
21
+ return {str(k): clean(x) for k, x in v.items()}
22
+ if isinstance(v, (list, tuple)):
23
+ return [clean(x) for x in v]
24
+ if v is None or isinstance(v, (str, int, float, bool)):
25
+ return v
26
+ if hasattr(v, "item"): # numpy / torch scalar
27
+ try:
28
+ return clean(v.item())
29
+ except Exception:
30
+ pass
31
+ return str(v)
32
+
33
+
34
+ def _stream(head, path, tail):
35
+ yield head
36
+ with open(path, "rb") as f:
37
+ while True:
38
+ chunk = f.read(1 << 20)
39
+ if not chunk:
40
+ break
41
+ yield chunk
42
+ yield tail
43
+
44
+
45
+ def request(method, url, key="", body=None, file=None, timeout=10):
46
+ """Sends body as JSON, or as multipart fields when file=(filename, bytes or Path) is given; a Path is streamed from
47
+ disk, so a large checkpoint never sits in memory. Returns the decoded reply."""
48
+ headers = {"Authorization": "Bearer " + key} if key else {}
49
+ data = None
50
+ if file is not None:
51
+ boundary = uuid.uuid4().hex
52
+ parts = []
53
+ for k, v in (body or {}).items():
54
+ parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode())
55
+ parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{file[0]}"\r\n'
56
+ f"Content-Type: application/octet-stream\r\n\r\n".encode())
57
+ head, tail = b"".join(parts), f"\r\n--{boundary}--\r\n".encode()
58
+ if isinstance(file[1], Path):
59
+ data = _stream(head, file[1], tail)
60
+ headers["Content-Length"] = str(len(head) + file[1].stat().st_size + len(tail))
61
+ else:
62
+ data = head + file[1] + tail
63
+ headers["Content-Type"] = "multipart/form-data; boundary=" + boundary
64
+ elif body is not None:
65
+ data = json.dumps(clean(body), allow_nan=False).encode()
66
+ headers["Content-Type"] = "application/json"
67
+ req = urllib.request.Request(url, data=data, method=method, headers=headers)
68
+ try:
69
+ with urllib.request.urlopen(req, timeout=timeout) as r:
70
+ raw = r.read()
71
+ except urllib.error.HTTPError as e:
72
+ raise HTTPError(e.code, e.read().decode(errors="replace")) from None
73
+ return json.loads(raw) if raw else None