runyard 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runyard/__init__.py +63 -0
- runyard/cli.py +219 -0
- runyard/client.py +73 -0
- runyard/core.py +485 -0
- runyard/integrations/__init__.py +2 -0
- runyard/integrations/accelerate.py +40 -0
- runyard/integrations/hf.py +51 -0
- runyard/integrations/keras.py +37 -0
- runyard/integrations/lightgbm.py +22 -0
- runyard/integrations/lightning.py +52 -0
- runyard/integrations/sklearn.py +61 -0
- runyard/integrations/torchtune.py +33 -0
- runyard/integrations/xgboost.py +45 -0
- runyard/media.py +95 -0
- runyard/wandb_hook.py +55 -0
- runyard/wandb_shim.py +256 -0
- runyard/watch.py +102 -0
- runyard-0.3.0.dist-info/METADATA +50 -0
- runyard-0.3.0.dist-info/RECORD +22 -0
- runyard-0.3.0.dist-info/WHEEL +4 -0
- runyard-0.3.0.dist-info/entry_points.txt +2 -0
- runyard_wandb.pth +1 -0
runyard/core.py
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
"""A run queues every call and sends it from a background thread, so a slow or missing server never blocks training.
|
|
2
|
+
|
|
3
|
+
Nothing here raises into the training script: failures become a warning on the real stderr.
|
|
4
|
+
"""
|
|
5
|
+
import atexit
|
|
6
|
+
import functools
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
import shutil
|
|
12
|
+
import signal
|
|
13
|
+
import socket
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
import uuid
|
|
19
|
+
from collections import deque
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from . import media
|
|
24
|
+
from .client import HTTPError, request
|
|
25
|
+
|
|
26
|
+
CACHE = Path(os.environ.get("RUNYARD_CACHE") or Path.home() / ".cache" / "runyard")
|
|
27
|
+
HEARTBEAT_EVERY = 30
|
|
28
|
+
SYSTEM_EVERY = 15
|
|
29
|
+
BATCH = 5000
|
|
30
|
+
|
|
31
|
+
_active = [] # open runs: the output tee and the exit hooks act on these
|
|
32
|
+
_warned = set()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _now():
|
|
36
|
+
return datetime.now(timezone.utc).isoformat()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def warn(msg, once=False):
|
|
40
|
+
if once:
|
|
41
|
+
if msg in _warned:
|
|
42
|
+
return
|
|
43
|
+
_warned.add(msg)
|
|
44
|
+
err = sys.__stderr__ # the real stderr, so a warning never turns into a captured log line
|
|
45
|
+
if err:
|
|
46
|
+
print("runyard: " + msg, file=err, flush=True)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _safe(f):
|
|
50
|
+
@functools.wraps(f)
|
|
51
|
+
def g(*a, **kw):
|
|
52
|
+
try:
|
|
53
|
+
return f(*a, **kw)
|
|
54
|
+
except Exception as e:
|
|
55
|
+
warn(f"{f.__name__} failed: {e!r}")
|
|
56
|
+
|
|
57
|
+
return g
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _number(v):
|
|
61
|
+
"""A JSON-ready metric value, or None for anything that is not a number."""
|
|
62
|
+
if hasattr(v, "item"): # numpy / torch scalar
|
|
63
|
+
try:
|
|
64
|
+
v = v.item()
|
|
65
|
+
except Exception:
|
|
66
|
+
return None
|
|
67
|
+
if isinstance(v, bool):
|
|
68
|
+
return int(v)
|
|
69
|
+
if not isinstance(v, (int, float)):
|
|
70
|
+
return None
|
|
71
|
+
if isinstance(v, float) and not math.isfinite(v):
|
|
72
|
+
return "NaN" if math.isnan(v) else ("Infinity" if v > 0 else "-Infinity")
|
|
73
|
+
return v
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Sender:
|
|
77
|
+
"""The ordered queue of API calls for one run. drain() sends from the head and stops at the first failure."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, server, key, run_id, items=()):
|
|
80
|
+
self.server, self.key, self.run_id = server, key, run_id
|
|
81
|
+
self.items = deque(items)
|
|
82
|
+
self.lock = threading.Lock()
|
|
83
|
+
|
|
84
|
+
# shortcut: the queue only lives in memory until exit; a server down for days on a chatty run grows it unbounded.
|
|
85
|
+
def put(self, kind, data):
|
|
86
|
+
with self.lock:
|
|
87
|
+
self.items.append((kind, data))
|
|
88
|
+
|
|
89
|
+
def drain(self):
|
|
90
|
+
"""True once the queue is empty, False when the server could not take the next call (it stays queued)."""
|
|
91
|
+
while True:
|
|
92
|
+
with self.lock:
|
|
93
|
+
if not self.items:
|
|
94
|
+
return True
|
|
95
|
+
kind = self.items[0][0]
|
|
96
|
+
n = 1
|
|
97
|
+
if kind in ("metrics", "logs"):
|
|
98
|
+
while n < len(self.items) and n < BATCH and self.items[n][0] == kind:
|
|
99
|
+
n += 1
|
|
100
|
+
batch = [self.items[i][1] for i in range(n)]
|
|
101
|
+
try:
|
|
102
|
+
self._send(kind, batch)
|
|
103
|
+
except HTTPError as e:
|
|
104
|
+
if e.status in (401, 403, 408, 429) or e.status >= 500:
|
|
105
|
+
if e.status in (401, 403):
|
|
106
|
+
warn(f"the server refused the API key ({e.status}); data is kept and saved at exit", once=True)
|
|
107
|
+
return False
|
|
108
|
+
warn(f"the server rejected {kind} ({e.status}), dropped: {e.body[:200]}")
|
|
109
|
+
except (OSError, ValueError): # unreachable, timed out, or a reply that is not JSON
|
|
110
|
+
return False
|
|
111
|
+
with self.lock:
|
|
112
|
+
for _ in range(n):
|
|
113
|
+
k, data = self.items.popleft()
|
|
114
|
+
if k == "media":
|
|
115
|
+
Path(data[1]).unlink(missing_ok=True)
|
|
116
|
+
|
|
117
|
+
def _send(self, kind, batch):
|
|
118
|
+
base = f"{self.server}/api/v1/runs"
|
|
119
|
+
if kind == "create":
|
|
120
|
+
request("POST", base, self.key, batch[0])
|
|
121
|
+
elif kind == "patch":
|
|
122
|
+
request("PATCH", f"{base}/{self.run_id}", self.key, batch[0])
|
|
123
|
+
elif kind in ("metrics", "logs"):
|
|
124
|
+
request("POST", f"{base}/{self.run_id}/{kind}", self.key, batch)
|
|
125
|
+
elif kind == "alert":
|
|
126
|
+
request("POST", f"{base}/{self.run_id}/alerts", self.key, batch[0])
|
|
127
|
+
elif kind == "media":
|
|
128
|
+
fields, path = batch[0]
|
|
129
|
+
if not Path(path).exists():
|
|
130
|
+
return # already sent by an earlier process (runyard sync after a partial upload)
|
|
131
|
+
request("POST", f"{base}/{self.run_id}/media", self.key, fields, file=(Path(path).name, Path(path)), timeout=300)
|
|
132
|
+
elif kind == "artifact":
|
|
133
|
+
fields, path = batch[0]
|
|
134
|
+
url = f"{base}/{self.run_id}/artifacts"
|
|
135
|
+
if path is None:
|
|
136
|
+
request("POST", url, self.key, {**fields, "meta": json.loads(fields["meta"])})
|
|
137
|
+
return
|
|
138
|
+
p = Path(path)
|
|
139
|
+
if not p.exists():
|
|
140
|
+
warn(f"artifact {path} is gone, not uploaded")
|
|
141
|
+
return
|
|
142
|
+
if p.is_file():
|
|
143
|
+
request("POST", url, self.key, fields, file=(p.name, p), timeout=3600)
|
|
144
|
+
return
|
|
145
|
+
# a directory goes up as one .tar.gz, packed here in the sender thread so training does not wait
|
|
146
|
+
archive = Path(shutil.make_archive(str(CACHE / "uploads" / fields["id"]), "gztar", p))
|
|
147
|
+
try:
|
|
148
|
+
request("POST", url, self.key, {**fields, "name": fields["name"] + ".tar.gz"}, file=(archive.name, archive), timeout=3600)
|
|
149
|
+
finally:
|
|
150
|
+
archive.unlink(missing_ok=True)
|
|
151
|
+
|
|
152
|
+
def save_pending(self):
|
|
153
|
+
"""Writes what is still queued to the cache so `runyard sync` can send it later; returns the file or None."""
|
|
154
|
+
with self.lock:
|
|
155
|
+
items = list(self.items)
|
|
156
|
+
if not items:
|
|
157
|
+
return None
|
|
158
|
+
p = CACHE / "pending" / f"{self.run_id}.json"
|
|
159
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
160
|
+
p.write_text(json.dumps({"server": self.server, "run_id": self.run_id, "items": items}))
|
|
161
|
+
return p
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class _Synced(dict):
|
|
165
|
+
"""A dict whose changes go to the server too (run.config, run.summary)."""
|
|
166
|
+
|
|
167
|
+
def __init__(self, run, field, data):
|
|
168
|
+
super().__init__(data)
|
|
169
|
+
self._run, self._field = run, field
|
|
170
|
+
|
|
171
|
+
def __setitem__(self, k, v):
|
|
172
|
+
super().__setitem__(k, v)
|
|
173
|
+
self._run._patch({self._field: {k: v}})
|
|
174
|
+
|
|
175
|
+
def update(self, *a, **kw):
|
|
176
|
+
d = dict(*a, **kw)
|
|
177
|
+
super().update(d)
|
|
178
|
+
self._run._patch({self._field: d})
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class Run:
|
|
182
|
+
def __init__(self, project=None, name=None, config=None, tags=None, notes="", id=None, server=None,
|
|
183
|
+
api_key=None, capture_output=True, system=True):
|
|
184
|
+
self.id = id or str(uuid.uuid4())
|
|
185
|
+
self.project = project or os.environ.get("RUNYARD_PROJECT") or "default"
|
|
186
|
+
self.name = name or os.environ.get("RUNYARD_RUN_NAME") or self.id[:8]
|
|
187
|
+
server = (server or os.environ.get("RUNYARD_SERVER") or "http://localhost:8080").rstrip("/")
|
|
188
|
+
self.url = f"{server}/#/r/{self.id}"
|
|
189
|
+
key = api_key or os.environ.get("RUNYARD_API_KEY", "")
|
|
190
|
+
if not key:
|
|
191
|
+
warn("RUNYARD_API_KEY is not set; the server will refuse this run", once=True)
|
|
192
|
+
self._sender = Sender(server, key, self.id)
|
|
193
|
+
self._dir = CACHE / "media" / self.id
|
|
194
|
+
self._step = 0
|
|
195
|
+
self._seq = 0
|
|
196
|
+
self._lock = threading.Lock()
|
|
197
|
+
self._capture = capture_output
|
|
198
|
+
self._closing = False
|
|
199
|
+
self._wake = threading.Event()
|
|
200
|
+
self.finished = False
|
|
201
|
+
self.config = _Synced(self, "config", config or {})
|
|
202
|
+
self.summary = _Synced(self, "summary", {})
|
|
203
|
+
self._sender.put("create", {"id": self.id, "project": self.project, "name": self.name, "config": dict(self.config),
|
|
204
|
+
"tags": list(tags or []), "notes": notes, "host": socket.gethostname(), "git": _git(),
|
|
205
|
+
"started_at": _now()})
|
|
206
|
+
self._system = _System() if system else None
|
|
207
|
+
self._t0 = time.time()
|
|
208
|
+
_active.append(self)
|
|
209
|
+
_install_hooks()
|
|
210
|
+
self._thread = threading.Thread(target=self._loop, name="runyard-sender", daemon=True)
|
|
211
|
+
self._thread.start()
|
|
212
|
+
|
|
213
|
+
def _patch(self, body):
|
|
214
|
+
if not self.finished:
|
|
215
|
+
self._sender.put("patch", body)
|
|
216
|
+
|
|
217
|
+
def _step_for(self, step):
|
|
218
|
+
"""The step for media logged without one: the last metrics step."""
|
|
219
|
+
return int(step) if step is not None else max(self._step - 1, 0)
|
|
220
|
+
|
|
221
|
+
@_safe
|
|
222
|
+
def log(self, data, step=None, ts=None):
|
|
223
|
+
"""Logs numbers. Without step, each call is the next step after the last one; ts (ISO 8601) defaults to now."""
|
|
224
|
+
values = {}
|
|
225
|
+
for k, v in data.items():
|
|
226
|
+
n = _number(v)
|
|
227
|
+
if n is None:
|
|
228
|
+
warn(f"skipped {k!r}: log() takes numbers, use image()/video()/text() for the rest", once=True)
|
|
229
|
+
else:
|
|
230
|
+
values[str(k)] = n
|
|
231
|
+
with self._lock:
|
|
232
|
+
if step is None:
|
|
233
|
+
step = self._step
|
|
234
|
+
step = int(step)
|
|
235
|
+
self._step = max(self._step, step + 1)
|
|
236
|
+
if values:
|
|
237
|
+
self._sender.put("metrics", {"step": step, "ts": ts or _now(), "values": values})
|
|
238
|
+
|
|
239
|
+
def _media(self, key, kind, step, path, meta):
|
|
240
|
+
mid = path.stem
|
|
241
|
+
self._sender.put("media", ({"id": mid, "key": key, "step": self._step_for(step), "kind": kind,
|
|
242
|
+
"meta": json.dumps(meta)}, str(path)))
|
|
243
|
+
|
|
244
|
+
@_safe
|
|
245
|
+
def image(self, key, images, step=None, caption=None):
|
|
246
|
+
"""One image or a list: PIL images, numpy arrays, torch tensors or image file paths."""
|
|
247
|
+
many = isinstance(images, (list, tuple))
|
|
248
|
+
for i, im in enumerate(images if many else [images]):
|
|
249
|
+
cap = caption[i] if isinstance(caption, (list, tuple)) else caption
|
|
250
|
+
try:
|
|
251
|
+
path = media.save_image(im, self._dir / f"{uuid.uuid4()}.png")
|
|
252
|
+
except Exception as e:
|
|
253
|
+
warn(f"could not save image {key!r}: {e!r}")
|
|
254
|
+
continue
|
|
255
|
+
self._media(key, "image", step, path, {"caption": cap, "index": i} if many else {"caption": cap})
|
|
256
|
+
|
|
257
|
+
@_safe
|
|
258
|
+
def video(self, key, video, fps=30, step=None, caption=None):
|
|
259
|
+
"""A video file path, or frames (arrays, tensors, PIL images) that become an mp4 (a GIF without imageio-ffmpeg)."""
|
|
260
|
+
path = media.save_video(video, self._dir / f"{uuid.uuid4()}.mp4", fps)
|
|
261
|
+
self._media(key, "video", step, path, {"caption": caption, "fps": fps})
|
|
262
|
+
|
|
263
|
+
@_safe
|
|
264
|
+
def text(self, key, rows, step=None):
|
|
265
|
+
"""Text samples: a string, or a list of strings or dicts (e.g. prompt/output pairs), shown as a table."""
|
|
266
|
+
rows = [rows] if isinstance(rows, str) else list(rows)
|
|
267
|
+
rows = [r if isinstance(r, dict) else {"text": str(r)} for r in rows]
|
|
268
|
+
self._dir.mkdir(parents=True, exist_ok=True)
|
|
269
|
+
path = self._dir / f"{uuid.uuid4()}.json"
|
|
270
|
+
from .client import clean
|
|
271
|
+
|
|
272
|
+
path.write_text(json.dumps(clean(rows)))
|
|
273
|
+
self._media(key, "text", step, path, {"rows": len(rows)})
|
|
274
|
+
|
|
275
|
+
@_safe
|
|
276
|
+
def artifact(self, path, name=None, kind="file", step=None, upload=False, meta=None):
|
|
277
|
+
"""Records a file or directory the run produced: a checkpoint, a LoRA, an exported model. By default only where
|
|
278
|
+
it lives (this host and path) is recorded; upload=True also sends a copy (a directory as one .tar.gz)."""
|
|
279
|
+
p = Path(path).resolve()
|
|
280
|
+
if not p.exists():
|
|
281
|
+
warn(f"artifact {path} does not exist, not recorded")
|
|
282
|
+
return
|
|
283
|
+
size = p.stat().st_size if p.is_file() else sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
|
|
284
|
+
fields = {"id": str(uuid.uuid4()), "name": name or p.name, "kind": kind, "uri": str(p), "size": size,
|
|
285
|
+
"step": self._step_for(step), "meta": json.dumps(meta or {})}
|
|
286
|
+
self._sender.put("artifact", (fields, str(p) if upload else None))
|
|
287
|
+
|
|
288
|
+
@_safe
|
|
289
|
+
def alert(self, title, body="", level="info"):
|
|
290
|
+
"""Records an alert and notifies the project's channels. level: info, warn or error."""
|
|
291
|
+
self._sender.put("alert", {"title": str(title), "body": str(body), "level": level})
|
|
292
|
+
|
|
293
|
+
def _log_line(self, stream, line):
|
|
294
|
+
with self._lock:
|
|
295
|
+
self._seq += 1
|
|
296
|
+
seq = self._seq
|
|
297
|
+
self._sender.put("logs", {"seq": seq, "ts": _now(), "stream": stream, "line": line[:10000]})
|
|
298
|
+
|
|
299
|
+
@_safe
|
|
300
|
+
def finish(self, status="finished", timeout=60):
|
|
301
|
+
"""Marks the run finished (or failed) and waits up to timeout seconds for the queue to reach the server."""
|
|
302
|
+
if self.finished:
|
|
303
|
+
return
|
|
304
|
+
if self in _active:
|
|
305
|
+
_active.remove(self)
|
|
306
|
+
self._sender.put("patch", {"status": status})
|
|
307
|
+
self.finished = True
|
|
308
|
+
self._closing = True
|
|
309
|
+
self._wake.set()
|
|
310
|
+
self._thread.join(timeout)
|
|
311
|
+
p = self._sender.save_pending()
|
|
312
|
+
if p:
|
|
313
|
+
warn(f"server unreachable, {len(self._sender.items)} unsent calls saved to {p}; send them with `runyard sync`")
|
|
314
|
+
|
|
315
|
+
def _loop(self):
|
|
316
|
+
last_beat = last_sys = 0.0
|
|
317
|
+
fails = 0
|
|
318
|
+
while True:
|
|
319
|
+
self._wake.wait(1.0 if not fails else min(60, 2 ** fails))
|
|
320
|
+
self._wake.clear()
|
|
321
|
+
now = time.time()
|
|
322
|
+
if self._system and now - last_sys >= SYSTEM_EVERY and not self._closing:
|
|
323
|
+
last_sys = now
|
|
324
|
+
values = self._system.sample()
|
|
325
|
+
if values:
|
|
326
|
+
self._sender.put("metrics", {"step": int(now - self._t0), "ts": _now(), "values": values})
|
|
327
|
+
closing = self._closing # read before draining: finish() queues its last call before setting it
|
|
328
|
+
ok = self._sender.drain()
|
|
329
|
+
fails = 0 if ok else fails + 1
|
|
330
|
+
if closing:
|
|
331
|
+
return
|
|
332
|
+
if ok and now - last_beat >= HEARTBEAT_EVERY:
|
|
333
|
+
try:
|
|
334
|
+
request("POST", f"{self._sender.server}/api/v1/runs/{self.id}/heartbeat", self._sender.key)
|
|
335
|
+
last_beat = now
|
|
336
|
+
except Exception:
|
|
337
|
+
pass
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def sync(path, api_key=None):
|
|
341
|
+
"""Sends a pending file written at exit by a run that could not reach its server. True when all of it landed."""
|
|
342
|
+
d = json.loads(Path(path).read_text())
|
|
343
|
+
s = Sender(d["server"], api_key or os.environ.get("RUNYARD_API_KEY", ""), d["run_id"], [tuple(i) for i in d["items"]])
|
|
344
|
+
if s.drain():
|
|
345
|
+
Path(path).unlink()
|
|
346
|
+
return True
|
|
347
|
+
s.save_pending()
|
|
348
|
+
return False
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _git():
|
|
352
|
+
def git(*args):
|
|
353
|
+
return subprocess.run(["git", *args], capture_output=True, text=True, timeout=5).stdout.strip()
|
|
354
|
+
|
|
355
|
+
try:
|
|
356
|
+
commit = git("rev-parse", "HEAD")
|
|
357
|
+
if not commit:
|
|
358
|
+
return None
|
|
359
|
+
return {"commit": commit, "branch": git("rev-parse", "--abbrev-ref", "HEAD"), "dirty": bool(git("status", "--porcelain")),
|
|
360
|
+
"remote": git("config", "--get", "remote.origin.url")}
|
|
361
|
+
except Exception:
|
|
362
|
+
return None
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
class _System:
|
|
366
|
+
"""GPU (nvidia-ml-py) and CPU/RAM (psutil) numbers, each only when its package is installed."""
|
|
367
|
+
|
|
368
|
+
def __init__(self):
|
|
369
|
+
self.nvml = self.psutil = None
|
|
370
|
+
try:
|
|
371
|
+
import pynvml
|
|
372
|
+
|
|
373
|
+
pynvml.nvmlInit()
|
|
374
|
+
self.gpus = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(pynvml.nvmlDeviceGetCount())]
|
|
375
|
+
self.nvml = pynvml
|
|
376
|
+
except Exception:
|
|
377
|
+
pass
|
|
378
|
+
try:
|
|
379
|
+
import psutil
|
|
380
|
+
|
|
381
|
+
psutil.cpu_percent()
|
|
382
|
+
self.psutil = psutil
|
|
383
|
+
except Exception:
|
|
384
|
+
pass
|
|
385
|
+
|
|
386
|
+
def sample(self):
|
|
387
|
+
v = {}
|
|
388
|
+
try:
|
|
389
|
+
n = self.nvml
|
|
390
|
+
for i, h in enumerate(self.gpus if n else []):
|
|
391
|
+
v[f"sys/gpu{i}.util"] = n.nvmlDeviceGetUtilizationRates(h).gpu
|
|
392
|
+
v[f"sys/gpu{i}.mem_gb"] = n.nvmlDeviceGetMemoryInfo(h).used / 1e9
|
|
393
|
+
v[f"sys/gpu{i}.temp_c"] = n.nvmlDeviceGetTemperature(h, 0)
|
|
394
|
+
v[f"sys/gpu{i}.power_w"] = n.nvmlDeviceGetPowerUsage(h) / 1000
|
|
395
|
+
if self.psutil:
|
|
396
|
+
v["sys/cpu"] = self.psutil.cpu_percent()
|
|
397
|
+
v["sys/ram_gb"] = self.psutil.virtual_memory().used / 1e9
|
|
398
|
+
except Exception:
|
|
399
|
+
pass
|
|
400
|
+
return v
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class _Tee:
|
|
404
|
+
"""Wraps sys.stdout/stderr: everything still reaches the terminal, and whole lines go to the open runs."""
|
|
405
|
+
|
|
406
|
+
def __init__(self, stream, number):
|
|
407
|
+
self._stream, self._number = stream, number
|
|
408
|
+
self._buf = ""
|
|
409
|
+
self._buf_lock = threading.Lock()
|
|
410
|
+
|
|
411
|
+
def write(self, s):
|
|
412
|
+
n = self._stream.write(s)
|
|
413
|
+
try:
|
|
414
|
+
with self._buf_lock:
|
|
415
|
+
self._buf += s
|
|
416
|
+
if "\n" not in self._buf and len(self._buf) < 10000:
|
|
417
|
+
return n
|
|
418
|
+
*lines, self._buf = self._buf.split("\n")
|
|
419
|
+
for line in lines:
|
|
420
|
+
line = line.rstrip("\r").rsplit("\r", 1)[-1] # a progress bar redrawn with \r: keep its last state
|
|
421
|
+
for run in list(_active):
|
|
422
|
+
if run._capture:
|
|
423
|
+
run._log_line(self._number, line)
|
|
424
|
+
except Exception:
|
|
425
|
+
pass
|
|
426
|
+
return n
|
|
427
|
+
|
|
428
|
+
def __getattr__(self, name):
|
|
429
|
+
return getattr(self._stream, name)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
_hooked = False
|
|
433
|
+
_exit_code = [0]
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _install_hooks():
|
|
437
|
+
_install_tee()
|
|
438
|
+
global _hooked
|
|
439
|
+
if _hooked:
|
|
440
|
+
return
|
|
441
|
+
_hooked = True
|
|
442
|
+
_install_exit_hooks()
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _install_tee():
|
|
446
|
+
# Runs at every init: something may have replaced sys.stdout since the last run started.
|
|
447
|
+
for name, number in (("stdout", 1), ("stderr", 2)):
|
|
448
|
+
orig = getattr(sys, name)
|
|
449
|
+
if orig is not None and not isinstance(orig, _Tee):
|
|
450
|
+
tee = _Tee(orig, number)
|
|
451
|
+
setattr(sys, name, tee)
|
|
452
|
+
# Logging handlers made before init (transformers makes one at import) hold the bare stream.
|
|
453
|
+
loggers = [logging.getLogger()] + [lg for lg in logging.root.manager.loggerDict.values() if isinstance(lg, logging.Logger)]
|
|
454
|
+
for lg in loggers:
|
|
455
|
+
for h in lg.handlers:
|
|
456
|
+
if type(h) is logging.StreamHandler and h.stream is orig:
|
|
457
|
+
h.setStream(tee)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _install_exit_hooks():
|
|
461
|
+
prev_hook = sys.excepthook
|
|
462
|
+
|
|
463
|
+
def excepthook(t, v, tb):
|
|
464
|
+
prev_hook(t, v, tb) # print the traceback first, so the tee sends it along with the run
|
|
465
|
+
for r in list(_active):
|
|
466
|
+
r.finish("failed")
|
|
467
|
+
|
|
468
|
+
sys.excepthook = excepthook
|
|
469
|
+
|
|
470
|
+
orig_exit = sys.exit
|
|
471
|
+
|
|
472
|
+
def exit(code=0):
|
|
473
|
+
_exit_code[0] = code
|
|
474
|
+
orig_exit(code)
|
|
475
|
+
|
|
476
|
+
sys.exit = exit
|
|
477
|
+
atexit.register(_at_exit)
|
|
478
|
+
if threading.current_thread() is threading.main_thread() and signal.getsignal(signal.SIGTERM) == signal.SIG_DFL:
|
|
479
|
+
signal.signal(signal.SIGTERM, lambda *_: sys.exit(128 + signal.SIGTERM)) # so atexit runs and marks it failed
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _at_exit():
|
|
483
|
+
code = _exit_code[0]
|
|
484
|
+
for r in list(_active):
|
|
485
|
+
r.finish("finished" if code in (None, 0) else "failed")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Accelerate: Accelerator(log_with=[RunyardTracker(project=...)]); accelerator.log() and init_trackers(config=...) then
|
|
2
|
+
write to runyard. The run starts when Accelerate first uses the tracker, on the main process only."""
|
|
3
|
+
from accelerate.tracking import GeneralTracker
|
|
4
|
+
|
|
5
|
+
from .. import init
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RunyardTracker(GeneralTracker):
|
|
9
|
+
name = "runyard"
|
|
10
|
+
requires_logging_directory = False
|
|
11
|
+
main_process_only = True
|
|
12
|
+
|
|
13
|
+
def __init__(self, project=None, run_name=None, **init_kwargs):
|
|
14
|
+
super().__init__()
|
|
15
|
+
self.project, self.run_name, self.init_kwargs = project, run_name, init_kwargs
|
|
16
|
+
self.run = None
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def tracker(self):
|
|
20
|
+
"""The runyard Run, which accelerator.get_tracker("runyard", unwrap=True) returns."""
|
|
21
|
+
if self.run is None:
|
|
22
|
+
self.run = init(self.project, self.run_name, **self.init_kwargs)
|
|
23
|
+
return self.run
|
|
24
|
+
|
|
25
|
+
def start(self):
|
|
26
|
+
self.tracker
|
|
27
|
+
|
|
28
|
+
def store_init_configuration(self, values):
|
|
29
|
+
self.tracker.config.update(values)
|
|
30
|
+
|
|
31
|
+
def log(self, values, step=None, **kwargs):
|
|
32
|
+
self.tracker.log(values, step=step)
|
|
33
|
+
|
|
34
|
+
def log_images(self, values, step=None, **kwargs):
|
|
35
|
+
for k, images in values.items():
|
|
36
|
+
self.tracker.image(k, list(images) if isinstance(images, (list, tuple)) else images, step=step)
|
|
37
|
+
|
|
38
|
+
def finish(self):
|
|
39
|
+
if self.run is not None:
|
|
40
|
+
self.run.finish()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Hugging Face Trainer (and TRL, Unsloth, which use it): Trainer(..., callbacks=[RunyardCallback()])."""
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from transformers import TrainerCallback
|
|
5
|
+
|
|
6
|
+
from .. import init
|
|
7
|
+
|
|
8
|
+
# what Trainer logs once at the end; they go to the run's summary, as W&B does
|
|
9
|
+
SUMMARY = ("train_runtime", "train_samples_per_second", "train_steps_per_second", "train_loss", "total_flos")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _key(k):
|
|
13
|
+
for p in ("eval_", "test_"):
|
|
14
|
+
if k.startswith(p):
|
|
15
|
+
return p[:-1] + "/" + k[len(p):]
|
|
16
|
+
return "train/" + k
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class RunyardCallback(TrainerCallback):
|
|
20
|
+
"""One run per training. Config is the TrainingArguments plus the model config; metrics are keyed by global step.
|
|
21
|
+
|
|
22
|
+
project and name default to RUNYARD_PROJECT and the Trainer's run_name; other keyword arguments go to runyard.init.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, project=None, name=None, **init_kwargs):
|
|
26
|
+
self.project, self.name, self.init_kwargs = project, name, init_kwargs
|
|
27
|
+
self.run = None
|
|
28
|
+
|
|
29
|
+
def on_train_begin(self, args, state, control, model=None, **kwargs):
|
|
30
|
+
if self.run is not None or not state.is_world_process_zero:
|
|
31
|
+
return
|
|
32
|
+
config = args.to_dict()
|
|
33
|
+
mc = getattr(model, "config", None)
|
|
34
|
+
if mc is not None:
|
|
35
|
+
config["model"] = mc if isinstance(mc, dict) else mc.to_dict()
|
|
36
|
+
name = self.name or (args.run_name if args.run_name and args.run_name != args.output_dir else None)
|
|
37
|
+
self.run = init(self.project, name, config, **self.init_kwargs)
|
|
38
|
+
|
|
39
|
+
def on_save(self, args, state, control, **kwargs):
|
|
40
|
+
# a record of where the checkpoint is (not a copy): runyard artifacts RUN lists them
|
|
41
|
+
path = os.path.join(args.output_dir, f"checkpoint-{state.global_step}")
|
|
42
|
+
if self.run is not None and os.path.isdir(path):
|
|
43
|
+
self.run.artifact(path, kind="checkpoint", step=state.global_step)
|
|
44
|
+
|
|
45
|
+
def on_log(self, args, state, control, logs=None, **kwargs):
|
|
46
|
+
if self.run is None or not logs:
|
|
47
|
+
return
|
|
48
|
+
self.run.summary.update({k: v for k, v in logs.items() if k in SUMMARY})
|
|
49
|
+
values = {_key(k): v for k, v in logs.items() if k not in SUMMARY}
|
|
50
|
+
if values:
|
|
51
|
+
self.run.log(values, step=state.global_step)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Keras: model.fit(..., callbacks=[RunyardCallback(project="mnist")])."""
|
|
2
|
+
import keras
|
|
3
|
+
|
|
4
|
+
from .. import init
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _key(k):
|
|
8
|
+
return "val/" + k[4:] if k.startswith("val_") else "train/" + k
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RunyardCallback(keras.callbacks.Callback):
|
|
12
|
+
"""One run per fit. Config is the fit parameters (epochs, steps) and the model's parameter count; epoch metrics
|
|
13
|
+
are keyed train/... and val/... at step=epoch. log_every_n_batches > 0 also logs batch/... every n batches, at the
|
|
14
|
+
global batch number. Other keyword arguments go to runyard.init.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, project=None, name=None, log_every_n_batches=0, **init_kwargs):
|
|
18
|
+
super().__init__()
|
|
19
|
+
self.project, self.name, self.every, self.init_kwargs = project, name, log_every_n_batches, init_kwargs
|
|
20
|
+
self.run = None
|
|
21
|
+
|
|
22
|
+
def on_train_begin(self, logs=None):
|
|
23
|
+
self.run = init(self.project, self.name, **self.init_kwargs)
|
|
24
|
+
self.run.config.update({**self.params, "model_params": self.model.count_params()})
|
|
25
|
+
self.batches = 0
|
|
26
|
+
|
|
27
|
+
def on_train_batch_end(self, batch, logs=None):
|
|
28
|
+
if self.every and logs and self.batches % self.every == 0:
|
|
29
|
+
self.run.log({"batch/" + k: v for k, v in logs.items()}, step=self.batches)
|
|
30
|
+
self.batches += 1
|
|
31
|
+
|
|
32
|
+
def on_epoch_end(self, epoch, logs=None):
|
|
33
|
+
if logs:
|
|
34
|
+
self.run.log({_key(k): v for k, v in logs.items()}, step=epoch)
|
|
35
|
+
|
|
36
|
+
def on_train_end(self, logs=None):
|
|
37
|
+
self.run.finish()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""LightGBM: lgb.train(params, dtrain, valid_sets=[dval], callbacks=[runyard_callback(project="tabular")])."""
|
|
2
|
+
from .. import init
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def runyard_callback(project=None, name=None, **init_kwargs):
|
|
6
|
+
"""A LightGBM callback: one run per training, config is the params, every eval result is logged as data/metric at
|
|
7
|
+
step=iteration. Other keyword arguments go to runyard.init.
|
|
8
|
+
"""
|
|
9
|
+
state = {}
|
|
10
|
+
|
|
11
|
+
def callback(env):
|
|
12
|
+
if env.iteration == env.begin_iteration:
|
|
13
|
+
state["run"] = init(project, name, dict(env.params), **init_kwargs)
|
|
14
|
+
run = state["run"]
|
|
15
|
+
# (data, metric, value, is_higher_better), plus stdv from lgb.cv
|
|
16
|
+
run.log({f"{r[0]}/{r[1]}": r[2] for r in env.evaluation_result_list}, step=env.iteration)
|
|
17
|
+
# shortcut: early stopping ends training before the last iteration, so that run is finished at exit instead;
|
|
18
|
+
# upgrade when someone needs the status right after training (wrap lgb.train, or catch EarlyStopException).
|
|
19
|
+
if env.iteration == env.end_iteration - 1:
|
|
20
|
+
run.finish()
|
|
21
|
+
|
|
22
|
+
return callback
|