runyard 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
runyard/core.py ADDED
@@ -0,0 +1,485 @@
1
+ """A run queues every call and sends it from a background thread, so a slow or missing server never blocks training.
2
+
3
+ Nothing here raises into the training script: failures become a warning on the real stderr.
4
+ """
5
+ import atexit
6
+ import functools
7
+ import json
8
+ import logging
9
+ import math
10
+ import os
11
+ import shutil
12
+ import signal
13
+ import socket
14
+ import subprocess
15
+ import sys
16
+ import threading
17
+ import time
18
+ import uuid
19
+ from collections import deque
20
+ from datetime import datetime, timezone
21
+ from pathlib import Path
22
+
23
+ from . import media
24
+ from .client import HTTPError, request
25
+
26
+ CACHE = Path(os.environ.get("RUNYARD_CACHE") or Path.home() / ".cache" / "runyard")
27
+ HEARTBEAT_EVERY = 30
28
+ SYSTEM_EVERY = 15
29
+ BATCH = 5000
30
+
31
+ _active = [] # open runs: the output tee and the exit hooks act on these
32
+ _warned = set()
33
+
34
+
35
+ def _now():
36
+ return datetime.now(timezone.utc).isoformat()
37
+
38
+
39
+ def warn(msg, once=False):
40
+ if once:
41
+ if msg in _warned:
42
+ return
43
+ _warned.add(msg)
44
+ err = sys.__stderr__ # the real stderr, so a warning never turns into a captured log line
45
+ if err:
46
+ print("runyard: " + msg, file=err, flush=True)
47
+
48
+
49
+ def _safe(f):
50
+ @functools.wraps(f)
51
+ def g(*a, **kw):
52
+ try:
53
+ return f(*a, **kw)
54
+ except Exception as e:
55
+ warn(f"{f.__name__} failed: {e!r}")
56
+
57
+ return g
58
+
59
+
60
+ def _number(v):
61
+ """A JSON-ready metric value, or None for anything that is not a number."""
62
+ if hasattr(v, "item"): # numpy / torch scalar
63
+ try:
64
+ v = v.item()
65
+ except Exception:
66
+ return None
67
+ if isinstance(v, bool):
68
+ return int(v)
69
+ if not isinstance(v, (int, float)):
70
+ return None
71
+ if isinstance(v, float) and not math.isfinite(v):
72
+ return "NaN" if math.isnan(v) else ("Infinity" if v > 0 else "-Infinity")
73
+ return v
74
+
75
+
76
+ class Sender:
77
+ """The ordered queue of API calls for one run. drain() sends from the head and stops at the first failure."""
78
+
79
+ def __init__(self, server, key, run_id, items=()):
80
+ self.server, self.key, self.run_id = server, key, run_id
81
+ self.items = deque(items)
82
+ self.lock = threading.Lock()
83
+
84
+ # shortcut: the queue only lives in memory until exit; a server down for days on a chatty run grows it unbounded.
85
+ def put(self, kind, data):
86
+ with self.lock:
87
+ self.items.append((kind, data))
88
+
89
+ def drain(self):
90
+ """True once the queue is empty, False when the server could not take the next call (it stays queued)."""
91
+ while True:
92
+ with self.lock:
93
+ if not self.items:
94
+ return True
95
+ kind = self.items[0][0]
96
+ n = 1
97
+ if kind in ("metrics", "logs"):
98
+ while n < len(self.items) and n < BATCH and self.items[n][0] == kind:
99
+ n += 1
100
+ batch = [self.items[i][1] for i in range(n)]
101
+ try:
102
+ self._send(kind, batch)
103
+ except HTTPError as e:
104
+ if e.status in (401, 403, 408, 429) or e.status >= 500:
105
+ if e.status in (401, 403):
106
+ warn(f"the server refused the API key ({e.status}); data is kept and saved at exit", once=True)
107
+ return False
108
+ warn(f"the server rejected {kind} ({e.status}), dropped: {e.body[:200]}")
109
+ except (OSError, ValueError): # unreachable, timed out, or a reply that is not JSON
110
+ return False
111
+ with self.lock:
112
+ for _ in range(n):
113
+ k, data = self.items.popleft()
114
+ if k == "media":
115
+ Path(data[1]).unlink(missing_ok=True)
116
+
117
+ def _send(self, kind, batch):
118
+ base = f"{self.server}/api/v1/runs"
119
+ if kind == "create":
120
+ request("POST", base, self.key, batch[0])
121
+ elif kind == "patch":
122
+ request("PATCH", f"{base}/{self.run_id}", self.key, batch[0])
123
+ elif kind in ("metrics", "logs"):
124
+ request("POST", f"{base}/{self.run_id}/{kind}", self.key, batch)
125
+ elif kind == "alert":
126
+ request("POST", f"{base}/{self.run_id}/alerts", self.key, batch[0])
127
+ elif kind == "media":
128
+ fields, path = batch[0]
129
+ if not Path(path).exists():
130
+ return # already sent by an earlier process (runyard sync after a partial upload)
131
+ request("POST", f"{base}/{self.run_id}/media", self.key, fields, file=(Path(path).name, Path(path)), timeout=300)
132
+ elif kind == "artifact":
133
+ fields, path = batch[0]
134
+ url = f"{base}/{self.run_id}/artifacts"
135
+ if path is None:
136
+ request("POST", url, self.key, {**fields, "meta": json.loads(fields["meta"])})
137
+ return
138
+ p = Path(path)
139
+ if not p.exists():
140
+ warn(f"artifact {path} is gone, not uploaded")
141
+ return
142
+ if p.is_file():
143
+ request("POST", url, self.key, fields, file=(p.name, p), timeout=3600)
144
+ return
145
+ # a directory goes up as one .tar.gz, packed here in the sender thread so training does not wait
146
+ archive = Path(shutil.make_archive(str(CACHE / "uploads" / fields["id"]), "gztar", p))
147
+ try:
148
+ request("POST", url, self.key, {**fields, "name": fields["name"] + ".tar.gz"}, file=(archive.name, archive), timeout=3600)
149
+ finally:
150
+ archive.unlink(missing_ok=True)
151
+
152
+ def save_pending(self):
153
+ """Writes what is still queued to the cache so `runyard sync` can send it later; returns the file or None."""
154
+ with self.lock:
155
+ items = list(self.items)
156
+ if not items:
157
+ return None
158
+ p = CACHE / "pending" / f"{self.run_id}.json"
159
+ p.parent.mkdir(parents=True, exist_ok=True)
160
+ p.write_text(json.dumps({"server": self.server, "run_id": self.run_id, "items": items}))
161
+ return p
162
+
163
+
164
+ class _Synced(dict):
165
+ """A dict whose changes go to the server too (run.config, run.summary)."""
166
+
167
+ def __init__(self, run, field, data):
168
+ super().__init__(data)
169
+ self._run, self._field = run, field
170
+
171
+ def __setitem__(self, k, v):
172
+ super().__setitem__(k, v)
173
+ self._run._patch({self._field: {k: v}})
174
+
175
+ def update(self, *a, **kw):
176
+ d = dict(*a, **kw)
177
+ super().update(d)
178
+ self._run._patch({self._field: d})
179
+
180
+
181
+ class Run:
182
+ def __init__(self, project=None, name=None, config=None, tags=None, notes="", id=None, server=None,
183
+ api_key=None, capture_output=True, system=True):
184
+ self.id = id or str(uuid.uuid4())
185
+ self.project = project or os.environ.get("RUNYARD_PROJECT") or "default"
186
+ self.name = name or os.environ.get("RUNYARD_RUN_NAME") or self.id[:8]
187
+ server = (server or os.environ.get("RUNYARD_SERVER") or "http://localhost:8080").rstrip("/")
188
+ self.url = f"{server}/#/r/{self.id}"
189
+ key = api_key or os.environ.get("RUNYARD_API_KEY", "")
190
+ if not key:
191
+ warn("RUNYARD_API_KEY is not set; the server will refuse this run", once=True)
192
+ self._sender = Sender(server, key, self.id)
193
+ self._dir = CACHE / "media" / self.id
194
+ self._step = 0
195
+ self._seq = 0
196
+ self._lock = threading.Lock()
197
+ self._capture = capture_output
198
+ self._closing = False
199
+ self._wake = threading.Event()
200
+ self.finished = False
201
+ self.config = _Synced(self, "config", config or {})
202
+ self.summary = _Synced(self, "summary", {})
203
+ self._sender.put("create", {"id": self.id, "project": self.project, "name": self.name, "config": dict(self.config),
204
+ "tags": list(tags or []), "notes": notes, "host": socket.gethostname(), "git": _git(),
205
+ "started_at": _now()})
206
+ self._system = _System() if system else None
207
+ self._t0 = time.time()
208
+ _active.append(self)
209
+ _install_hooks()
210
+ self._thread = threading.Thread(target=self._loop, name="runyard-sender", daemon=True)
211
+ self._thread.start()
212
+
213
+ def _patch(self, body):
214
+ if not self.finished:
215
+ self._sender.put("patch", body)
216
+
217
+ def _step_for(self, step):
218
+ """The step for media logged without one: the last metrics step."""
219
+ return int(step) if step is not None else max(self._step - 1, 0)
220
+
221
+ @_safe
222
+ def log(self, data, step=None, ts=None):
223
+ """Logs numbers. Without step, each call is the next step after the last one; ts (ISO 8601) defaults to now."""
224
+ values = {}
225
+ for k, v in data.items():
226
+ n = _number(v)
227
+ if n is None:
228
+ warn(f"skipped {k!r}: log() takes numbers, use image()/video()/text() for the rest", once=True)
229
+ else:
230
+ values[str(k)] = n
231
+ with self._lock:
232
+ if step is None:
233
+ step = self._step
234
+ step = int(step)
235
+ self._step = max(self._step, step + 1)
236
+ if values:
237
+ self._sender.put("metrics", {"step": step, "ts": ts or _now(), "values": values})
238
+
239
+ def _media(self, key, kind, step, path, meta):
240
+ mid = path.stem
241
+ self._sender.put("media", ({"id": mid, "key": key, "step": self._step_for(step), "kind": kind,
242
+ "meta": json.dumps(meta)}, str(path)))
243
+
244
+ @_safe
245
+ def image(self, key, images, step=None, caption=None):
246
+ """One image or a list: PIL images, numpy arrays, torch tensors or image file paths."""
247
+ many = isinstance(images, (list, tuple))
248
+ for i, im in enumerate(images if many else [images]):
249
+ cap = caption[i] if isinstance(caption, (list, tuple)) else caption
250
+ try:
251
+ path = media.save_image(im, self._dir / f"{uuid.uuid4()}.png")
252
+ except Exception as e:
253
+ warn(f"could not save image {key!r}: {e!r}")
254
+ continue
255
+ self._media(key, "image", step, path, {"caption": cap, "index": i} if many else {"caption": cap})
256
+
257
+ @_safe
258
+ def video(self, key, video, fps=30, step=None, caption=None):
259
+ """A video file path, or frames (arrays, tensors, PIL images) that become an mp4 (a GIF without imageio-ffmpeg)."""
260
+ path = media.save_video(video, self._dir / f"{uuid.uuid4()}.mp4", fps)
261
+ self._media(key, "video", step, path, {"caption": caption, "fps": fps})
262
+
263
+ @_safe
264
+ def text(self, key, rows, step=None):
265
+ """Text samples: a string, or a list of strings or dicts (e.g. prompt/output pairs), shown as a table."""
266
+ rows = [rows] if isinstance(rows, str) else list(rows)
267
+ rows = [r if isinstance(r, dict) else {"text": str(r)} for r in rows]
268
+ self._dir.mkdir(parents=True, exist_ok=True)
269
+ path = self._dir / f"{uuid.uuid4()}.json"
270
+ from .client import clean
271
+
272
+ path.write_text(json.dumps(clean(rows)))
273
+ self._media(key, "text", step, path, {"rows": len(rows)})
274
+
275
+ @_safe
276
+ def artifact(self, path, name=None, kind="file", step=None, upload=False, meta=None):
277
+ """Records a file or directory the run produced: a checkpoint, a LoRA, an exported model. By default only where
278
+ it lives (this host and path) is recorded; upload=True also sends a copy (a directory as one .tar.gz)."""
279
+ p = Path(path).resolve()
280
+ if not p.exists():
281
+ warn(f"artifact {path} does not exist, not recorded")
282
+ return
283
+ size = p.stat().st_size if p.is_file() else sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
284
+ fields = {"id": str(uuid.uuid4()), "name": name or p.name, "kind": kind, "uri": str(p), "size": size,
285
+ "step": self._step_for(step), "meta": json.dumps(meta or {})}
286
+ self._sender.put("artifact", (fields, str(p) if upload else None))
287
+
288
+ @_safe
289
+ def alert(self, title, body="", level="info"):
290
+ """Records an alert and notifies the project's channels. level: info, warn or error."""
291
+ self._sender.put("alert", {"title": str(title), "body": str(body), "level": level})
292
+
293
+ def _log_line(self, stream, line):
294
+ with self._lock:
295
+ self._seq += 1
296
+ seq = self._seq
297
+ self._sender.put("logs", {"seq": seq, "ts": _now(), "stream": stream, "line": line[:10000]})
298
+
299
+ @_safe
300
+ def finish(self, status="finished", timeout=60):
301
+ """Marks the run finished (or failed) and waits up to timeout seconds for the queue to reach the server."""
302
+ if self.finished:
303
+ return
304
+ if self in _active:
305
+ _active.remove(self)
306
+ self._sender.put("patch", {"status": status})
307
+ self.finished = True
308
+ self._closing = True
309
+ self._wake.set()
310
+ self._thread.join(timeout)
311
+ p = self._sender.save_pending()
312
+ if p:
313
+ warn(f"server unreachable, {len(self._sender.items)} unsent calls saved to {p}; send them with `runyard sync`")
314
+
315
+ def _loop(self):
316
+ last_beat = last_sys = 0.0
317
+ fails = 0
318
+ while True:
319
+ self._wake.wait(1.0 if not fails else min(60, 2 ** fails))
320
+ self._wake.clear()
321
+ now = time.time()
322
+ if self._system and now - last_sys >= SYSTEM_EVERY and not self._closing:
323
+ last_sys = now
324
+ values = self._system.sample()
325
+ if values:
326
+ self._sender.put("metrics", {"step": int(now - self._t0), "ts": _now(), "values": values})
327
+ closing = self._closing # read before draining: finish() queues its last call before setting it
328
+ ok = self._sender.drain()
329
+ fails = 0 if ok else fails + 1
330
+ if closing:
331
+ return
332
+ if ok and now - last_beat >= HEARTBEAT_EVERY:
333
+ try:
334
+ request("POST", f"{self._sender.server}/api/v1/runs/{self.id}/heartbeat", self._sender.key)
335
+ last_beat = now
336
+ except Exception:
337
+ pass
338
+
339
+
340
+ def sync(path, api_key=None):
341
+ """Sends a pending file written at exit by a run that could not reach its server. True when all of it landed."""
342
+ d = json.loads(Path(path).read_text())
343
+ s = Sender(d["server"], api_key or os.environ.get("RUNYARD_API_KEY", ""), d["run_id"], [tuple(i) for i in d["items"]])
344
+ if s.drain():
345
+ Path(path).unlink()
346
+ return True
347
+ s.save_pending()
348
+ return False
349
+
350
+
351
+ def _git():
352
+ def git(*args):
353
+ return subprocess.run(["git", *args], capture_output=True, text=True, timeout=5).stdout.strip()
354
+
355
+ try:
356
+ commit = git("rev-parse", "HEAD")
357
+ if not commit:
358
+ return None
359
+ return {"commit": commit, "branch": git("rev-parse", "--abbrev-ref", "HEAD"), "dirty": bool(git("status", "--porcelain")),
360
+ "remote": git("config", "--get", "remote.origin.url")}
361
+ except Exception:
362
+ return None
363
+
364
+
365
+ class _System:
366
+ """GPU (nvidia-ml-py) and CPU/RAM (psutil) numbers, each only when its package is installed."""
367
+
368
+ def __init__(self):
369
+ self.nvml = self.psutil = None
370
+ try:
371
+ import pynvml
372
+
373
+ pynvml.nvmlInit()
374
+ self.gpus = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(pynvml.nvmlDeviceGetCount())]
375
+ self.nvml = pynvml
376
+ except Exception:
377
+ pass
378
+ try:
379
+ import psutil
380
+
381
+ psutil.cpu_percent()
382
+ self.psutil = psutil
383
+ except Exception:
384
+ pass
385
+
386
+ def sample(self):
387
+ v = {}
388
+ try:
389
+ n = self.nvml
390
+ for i, h in enumerate(self.gpus if n else []):
391
+ v[f"sys/gpu{i}.util"] = n.nvmlDeviceGetUtilizationRates(h).gpu
392
+ v[f"sys/gpu{i}.mem_gb"] = n.nvmlDeviceGetMemoryInfo(h).used / 1e9
393
+ v[f"sys/gpu{i}.temp_c"] = n.nvmlDeviceGetTemperature(h, 0)
394
+ v[f"sys/gpu{i}.power_w"] = n.nvmlDeviceGetPowerUsage(h) / 1000
395
+ if self.psutil:
396
+ v["sys/cpu"] = self.psutil.cpu_percent()
397
+ v["sys/ram_gb"] = self.psutil.virtual_memory().used / 1e9
398
+ except Exception:
399
+ pass
400
+ return v
401
+
402
+
403
+ class _Tee:
404
+ """Wraps sys.stdout/stderr: everything still reaches the terminal, and whole lines go to the open runs."""
405
+
406
+ def __init__(self, stream, number):
407
+ self._stream, self._number = stream, number
408
+ self._buf = ""
409
+ self._buf_lock = threading.Lock()
410
+
411
+ def write(self, s):
412
+ n = self._stream.write(s)
413
+ try:
414
+ with self._buf_lock:
415
+ self._buf += s
416
+ if "\n" not in self._buf and len(self._buf) < 10000:
417
+ return n
418
+ *lines, self._buf = self._buf.split("\n")
419
+ for line in lines:
420
+ line = line.rstrip("\r").rsplit("\r", 1)[-1] # a progress bar redrawn with \r: keep its last state
421
+ for run in list(_active):
422
+ if run._capture:
423
+ run._log_line(self._number, line)
424
+ except Exception:
425
+ pass
426
+ return n
427
+
428
+ def __getattr__(self, name):
429
+ return getattr(self._stream, name)
430
+
431
+
432
+ _hooked = False
433
+ _exit_code = [0]
434
+
435
+
436
+ def _install_hooks():
437
+ _install_tee()
438
+ global _hooked
439
+ if _hooked:
440
+ return
441
+ _hooked = True
442
+ _install_exit_hooks()
443
+
444
+
445
+ def _install_tee():
446
+ # Runs at every init: something may have replaced sys.stdout since the last run started.
447
+ for name, number in (("stdout", 1), ("stderr", 2)):
448
+ orig = getattr(sys, name)
449
+ if orig is not None and not isinstance(orig, _Tee):
450
+ tee = _Tee(orig, number)
451
+ setattr(sys, name, tee)
452
+ # Logging handlers made before init (transformers makes one at import) hold the bare stream.
453
+ loggers = [logging.getLogger()] + [lg for lg in logging.root.manager.loggerDict.values() if isinstance(lg, logging.Logger)]
454
+ for lg in loggers:
455
+ for h in lg.handlers:
456
+ if type(h) is logging.StreamHandler and h.stream is orig:
457
+ h.setStream(tee)
458
+
459
+
460
+ def _install_exit_hooks():
461
+ prev_hook = sys.excepthook
462
+
463
+ def excepthook(t, v, tb):
464
+ prev_hook(t, v, tb) # print the traceback first, so the tee sends it along with the run
465
+ for r in list(_active):
466
+ r.finish("failed")
467
+
468
+ sys.excepthook = excepthook
469
+
470
+ orig_exit = sys.exit
471
+
472
+ def exit(code=0):
473
+ _exit_code[0] = code
474
+ orig_exit(code)
475
+
476
+ sys.exit = exit
477
+ atexit.register(_at_exit)
478
+ if threading.current_thread() is threading.main_thread() and signal.getsignal(signal.SIGTERM) == signal.SIG_DFL:
479
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(128 + signal.SIGTERM)) # so atexit runs and marks it failed
480
+
481
+
482
+ def _at_exit():
483
+ code = _exit_code[0]
484
+ for r in list(_active):
485
+ r.finish("finished" if code in (None, 0) else "failed")
@@ -0,0 +1,2 @@
1
+ """Adapters for training frameworks: hf (transformers, TRL, Unsloth), accelerate, lightning, keras, xgboost, lightgbm,
2
+ sklearn and torchtune. Each imports its framework."""
@@ -0,0 +1,40 @@
1
+ """Accelerate: Accelerator(log_with=[RunyardTracker(project=...)]); accelerator.log() and init_trackers(config=...) then
2
+ write to runyard. The run starts when Accelerate first uses the tracker, on the main process only."""
3
+ from accelerate.tracking import GeneralTracker
4
+
5
+ from .. import init
6
+
7
+
8
+ class RunyardTracker(GeneralTracker):
9
+ name = "runyard"
10
+ requires_logging_directory = False
11
+ main_process_only = True
12
+
13
+ def __init__(self, project=None, run_name=None, **init_kwargs):
14
+ super().__init__()
15
+ self.project, self.run_name, self.init_kwargs = project, run_name, init_kwargs
16
+ self.run = None
17
+
18
+ @property
19
+ def tracker(self):
20
+ """The runyard Run, which accelerator.get_tracker("runyard", unwrap=True) returns."""
21
+ if self.run is None:
22
+ self.run = init(self.project, self.run_name, **self.init_kwargs)
23
+ return self.run
24
+
25
+ def start(self):
26
+ self.tracker
27
+
28
+ def store_init_configuration(self, values):
29
+ self.tracker.config.update(values)
30
+
31
+ def log(self, values, step=None, **kwargs):
32
+ self.tracker.log(values, step=step)
33
+
34
+ def log_images(self, values, step=None, **kwargs):
35
+ for k, images in values.items():
36
+ self.tracker.image(k, list(images) if isinstance(images, (list, tuple)) else images, step=step)
37
+
38
+ def finish(self):
39
+ if self.run is not None:
40
+ self.run.finish()
@@ -0,0 +1,51 @@
1
+ """Hugging Face Trainer (and TRL, Unsloth, which use it): Trainer(..., callbacks=[RunyardCallback()])."""
2
+ import os
3
+
4
+ from transformers import TrainerCallback
5
+
6
+ from .. import init
7
+
8
+ # what Trainer logs once at the end; they go to the run's summary, as W&B does
9
+ SUMMARY = ("train_runtime", "train_samples_per_second", "train_steps_per_second", "train_loss", "total_flos")
10
+
11
+
12
+ def _key(k):
13
+ for p in ("eval_", "test_"):
14
+ if k.startswith(p):
15
+ return p[:-1] + "/" + k[len(p):]
16
+ return "train/" + k
17
+
18
+
19
+ class RunyardCallback(TrainerCallback):
20
+ """One run per training. Config is the TrainingArguments plus the model config; metrics are keyed by global step.
21
+
22
+ project and name default to RUNYARD_PROJECT and the Trainer's run_name; other keyword arguments go to runyard.init.
23
+ """
24
+
25
+ def __init__(self, project=None, name=None, **init_kwargs):
26
+ self.project, self.name, self.init_kwargs = project, name, init_kwargs
27
+ self.run = None
28
+
29
+ def on_train_begin(self, args, state, control, model=None, **kwargs):
30
+ if self.run is not None or not state.is_world_process_zero:
31
+ return
32
+ config = args.to_dict()
33
+ mc = getattr(model, "config", None)
34
+ if mc is not None:
35
+ config["model"] = mc if isinstance(mc, dict) else mc.to_dict()
36
+ name = self.name or (args.run_name if args.run_name and args.run_name != args.output_dir else None)
37
+ self.run = init(self.project, name, config, **self.init_kwargs)
38
+
39
+ def on_save(self, args, state, control, **kwargs):
40
+ # a record of where the checkpoint is (not a copy): runyard artifacts RUN lists them
41
+ path = os.path.join(args.output_dir, f"checkpoint-{state.global_step}")
42
+ if self.run is not None and os.path.isdir(path):
43
+ self.run.artifact(path, kind="checkpoint", step=state.global_step)
44
+
45
+ def on_log(self, args, state, control, logs=None, **kwargs):
46
+ if self.run is None or not logs:
47
+ return
48
+ self.run.summary.update({k: v for k, v in logs.items() if k in SUMMARY})
49
+ values = {_key(k): v for k, v in logs.items() if k not in SUMMARY}
50
+ if values:
51
+ self.run.log(values, step=state.global_step)
@@ -0,0 +1,37 @@
1
+ """Keras: model.fit(..., callbacks=[RunyardCallback(project="mnist")])."""
2
+ import keras
3
+
4
+ from .. import init
5
+
6
+
7
+ def _key(k):
8
+ return "val/" + k[4:] if k.startswith("val_") else "train/" + k
9
+
10
+
11
+ class RunyardCallback(keras.callbacks.Callback):
12
+ """One run per fit. Config is the fit parameters (epochs, steps) and the model's parameter count; epoch metrics
13
+ are keyed train/... and val/... at step=epoch. log_every_n_batches > 0 also logs batch/... every n batches, at the
14
+ global batch number. Other keyword arguments go to runyard.init.
15
+ """
16
+
17
+ def __init__(self, project=None, name=None, log_every_n_batches=0, **init_kwargs):
18
+ super().__init__()
19
+ self.project, self.name, self.every, self.init_kwargs = project, name, log_every_n_batches, init_kwargs
20
+ self.run = None
21
+
22
+ def on_train_begin(self, logs=None):
23
+ self.run = init(self.project, self.name, **self.init_kwargs)
24
+ self.run.config.update({**self.params, "model_params": self.model.count_params()})
25
+ self.batches = 0
26
+
27
+ def on_train_batch_end(self, batch, logs=None):
28
+ if self.every and logs and self.batches % self.every == 0:
29
+ self.run.log({"batch/" + k: v for k, v in logs.items()}, step=self.batches)
30
+ self.batches += 1
31
+
32
+ def on_epoch_end(self, epoch, logs=None):
33
+ if logs:
34
+ self.run.log({_key(k): v for k, v in logs.items()}, step=epoch)
35
+
36
+ def on_train_end(self, logs=None):
37
+ self.run.finish()
@@ -0,0 +1,22 @@
1
+ """LightGBM: lgb.train(params, dtrain, valid_sets=[dval], callbacks=[runyard_callback(project="tabular")])."""
2
+ from .. import init
3
+
4
+
5
+ def runyard_callback(project=None, name=None, **init_kwargs):
6
+ """A LightGBM callback: one run per training, config is the params, every eval result is logged as data/metric at
7
+ step=iteration. Other keyword arguments go to runyard.init.
8
+ """
9
+ state = {}
10
+
11
+ def callback(env):
12
+ if env.iteration == env.begin_iteration:
13
+ state["run"] = init(project, name, dict(env.params), **init_kwargs)
14
+ run = state["run"]
15
+ # (data, metric, value, is_higher_better), plus stdv from lgb.cv
16
+ run.log({f"{r[0]}/{r[1]}": r[2] for r in env.evaluation_result_list}, step=env.iteration)
17
+ # shortcut: early stopping ends training before the last iteration, so that run is finished at exit instead;
18
+ # upgrade when someone needs the status right after training (wrap lgb.train, or catch EarlyStopException).
19
+ if env.iteration == env.end_iteration - 1:
20
+ run.finish()
21
+
22
+ return callback