runyard 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runyard/__init__.py +63 -0
- runyard/cli.py +219 -0
- runyard/client.py +73 -0
- runyard/core.py +485 -0
- runyard/integrations/__init__.py +2 -0
- runyard/integrations/accelerate.py +40 -0
- runyard/integrations/hf.py +51 -0
- runyard/integrations/keras.py +37 -0
- runyard/integrations/lightgbm.py +22 -0
- runyard/integrations/lightning.py +52 -0
- runyard/integrations/sklearn.py +61 -0
- runyard/integrations/torchtune.py +33 -0
- runyard/integrations/xgboost.py +45 -0
- runyard/media.py +95 -0
- runyard/wandb_hook.py +55 -0
- runyard/wandb_shim.py +256 -0
- runyard/watch.py +102 -0
- runyard-0.3.0.dist-info/METADATA +50 -0
- runyard-0.3.0.dist-info/RECORD +22 -0
- runyard-0.3.0.dist-info/WHEEL +4 -0
- runyard-0.3.0.dist-info/entry_points.txt +2 -0
- runyard_wandb.pth +1 -0
runyard/watch.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""`runyard watch DIR`: sends TensorBoard event files to runyard, for tools that write nothing else (kohya sd-scripts,
|
|
2
|
+
DeepSpeed, diffusers --report_to tensorboard, torch SummaryWriter). Needs the tensorboard package.
|
|
3
|
+
|
|
4
|
+
Each directory holding events.out.tfevents.* files is one run, named by its path under DIR. Scalars, images, text
|
|
5
|
+
and hparams are read; other summaries (histograms, audio, graphs) are skipped.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import socket
|
|
11
|
+
import time
|
|
12
|
+
import uuid
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from tensorboard.backend.event_processing.event_file_loader import EventFileLoader
|
|
17
|
+
from tensorboard.plugins.hparams import plugin_data_pb2
|
|
18
|
+
from tensorboard.util import tensor_util
|
|
19
|
+
|
|
20
|
+
from .core import Run, _safe, warn
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def run_dir(f, root):
|
|
24
|
+
"""An event file's run directory: its own, or the parent of torch's add_hparams subdirectory (<logdir>/<time>)."""
|
|
25
|
+
d = f.parent
|
|
26
|
+
return d.parent if d != root and re.fullmatch(r"\d{9,}\.\d+", d.name) else d
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def start(d, root, project, live):
|
|
30
|
+
name = d.relative_to(root).as_posix() if d != root else root.name
|
|
31
|
+
# The same directory on the same machine is always the same run, so a restarted watcher continues it.
|
|
32
|
+
rid = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{socket.gethostname()}:{d}"))
|
|
33
|
+
# shortcut: started_at is when the watcher first saw the directory, not the first event's time; upgrade by letting
|
|
34
|
+
# Run take started_at if imports of old trainings sort wrongly in the run list.
|
|
35
|
+
run = Run(project, name, id=rid, capture_output=False, system=False)
|
|
36
|
+
if live:
|
|
37
|
+
run._patch({"status": "running"}) # an earlier `watch --once` may have finished it
|
|
38
|
+
print(f"{name}: {run.url}", flush=True)
|
|
39
|
+
return run
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def save(run, kind, key, ev, i, data, ext, meta):
|
|
43
|
+
# A fixed id per sample: the server ignores the ones a restarted watcher reads again.
|
|
44
|
+
mid = uuid.uuid5(uuid.UUID(run.id), f"{key}/{ev.step}/{i}/{ev.wall_time}")
|
|
45
|
+
path = run._dir / f"{mid}{ext}"
|
|
46
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
path.write_bytes(data)
|
|
48
|
+
run._media(key, kind, ev.step, path, meta)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@_safe
|
|
52
|
+
def send(run, ev, plugins):
|
|
53
|
+
"""Sends one event's summaries. plugins maps tag to plugin name, which TF2 writers put on a tag's first value only."""
|
|
54
|
+
scalars = {}
|
|
55
|
+
for v in ev.summary.value:
|
|
56
|
+
plugin = plugins.setdefault(v.tag, v.metadata.plugin_data.plugin_name)
|
|
57
|
+
if plugin == "scalars":
|
|
58
|
+
scalars[v.tag] = tensor_util.make_ndarray(v.tensor).item()
|
|
59
|
+
elif plugin == "images":
|
|
60
|
+
imgs = v.tensor.string_val[2:] # width, height, then the encoded images
|
|
61
|
+
for i, b in enumerate(imgs):
|
|
62
|
+
ext = ".png" if b[:4] == b"\x89PNG" else ".gif" if b[:4] == b"GIF8" else ".jpg"
|
|
63
|
+
meta = {"caption": None, "index": i} if len(imgs) > 1 else {"caption": None}
|
|
64
|
+
save(run, "image", v.tag, ev, i, b, ext, meta)
|
|
65
|
+
elif plugin == "text":
|
|
66
|
+
rows = [{"text": s.decode("utf-8", "replace")} for s in v.tensor.string_val]
|
|
67
|
+
key = v.tag.removesuffix("/text_summary") # torch's add_text appends it
|
|
68
|
+
save(run, "text", key, ev, 0, json.dumps(rows).encode(), ".json", {"rows": len(rows)})
|
|
69
|
+
elif plugin == "hparams":
|
|
70
|
+
hp = plugin_data_pb2.HParamsPluginData.FromString(v.metadata.plugin_data.content).session_start_info.hparams
|
|
71
|
+
if hp:
|
|
72
|
+
run.config.update({k: getattr(x, x.WhichOneof("kind")) for k, x in hp.items() if x.WhichOneof("kind")})
|
|
73
|
+
else:
|
|
74
|
+
warn(f"skipped {plugin or 'untyped'} summaries: watch reads scalars, images, text and hparams", once=True)
|
|
75
|
+
if scalars:
|
|
76
|
+
run.log(scalars, step=ev.step, ts=datetime.fromtimestamp(ev.wall_time, timezone.utc).isoformat())
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# shortcut: a run stays running while the watcher runs, as event files say nothing about a training's end; upgrade to
|
|
80
|
+
# finishing a run after some idle time if long-lived watchers over many finished trainings become common.
|
|
81
|
+
def watch(root, project, once=False, interval=10):
|
|
82
|
+
"""Reads every event file under root, then (unless once) polls for more until Ctrl-C; finishes the runs."""
|
|
83
|
+
root = Path(root).resolve()
|
|
84
|
+
project = project or os.environ.get("RUNYARD_PROJECT") or root.name
|
|
85
|
+
runs, loaders, plugins = {}, {}, {}
|
|
86
|
+
try:
|
|
87
|
+
while True:
|
|
88
|
+
for f in sorted(root.rglob("events.out.tfevents.*")):
|
|
89
|
+
d = run_dir(f, root)
|
|
90
|
+
if d not in runs:
|
|
91
|
+
runs[d] = start(d, root, project, live=not once)
|
|
92
|
+
if f not in loaders:
|
|
93
|
+
loaders[f] = EventFileLoader(str(f)) # keeps its place in the file, so polls read only new events
|
|
94
|
+
for ev in loaders[f].Load():
|
|
95
|
+
send(runs[d], ev, plugins.setdefault(d, {}))
|
|
96
|
+
if once:
|
|
97
|
+
break
|
|
98
|
+
time.sleep(interval)
|
|
99
|
+
except KeyboardInterrupt:
|
|
100
|
+
pass
|
|
101
|
+
for run in runs.values():
|
|
102
|
+
run.finish()
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: runyard
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Python client for runyard, a lightweight self-hosted training tracker
|
|
5
|
+
Project-URL: Homepage, https://github.com/cobanov/runyard
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: numpy; extra == 'dev'
|
|
10
|
+
Requires-Dist: pillow; extra == 'dev'
|
|
11
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
12
|
+
Provides-Extra: images
|
|
13
|
+
Requires-Dist: numpy; extra == 'images'
|
|
14
|
+
Requires-Dist: pillow; extra == 'images'
|
|
15
|
+
Provides-Extra: system
|
|
16
|
+
Requires-Dist: nvidia-ml-py; extra == 'system'
|
|
17
|
+
Requires-Dist: psutil; extra == 'system'
|
|
18
|
+
Provides-Extra: tensorboard
|
|
19
|
+
Requires-Dist: tensorboard; extra == 'tensorboard'
|
|
20
|
+
Provides-Extra: video
|
|
21
|
+
Requires-Dist: imageio; extra == 'video'
|
|
22
|
+
Requires-Dist: imageio-ffmpeg; extra == 'video'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# runyard (Python client)
|
|
26
|
+
|
|
27
|
+
See the [main README](https://github.com/cobanov/runyard).
|
|
28
|
+
|
|
29
|
+
Optional extras: `images` (numpy, Pillow), `system` (GPU, CPU and RAM usage), `video` (imageio-ffmpeg: frames
|
|
30
|
+
become an H.264 mp4 instead of a GIF) and `tensorboard` (for `runyard watch DIR`, which sends TensorBoard event
|
|
31
|
+
files: one run per directory, `--once` to import and exit). `runyard --help` lists the commands.
|
|
32
|
+
|
|
33
|
+
## Framework adapters
|
|
34
|
+
|
|
35
|
+
`runyard.integrations.<module>` imports its framework, which you install yourself. Keyword arguments other than
|
|
36
|
+
`project` and `name` go to `runyard.init` (`tags`, `notes`, `system`, ...).
|
|
37
|
+
|
|
38
|
+
| Module | One line | Run, config, metrics |
|
|
39
|
+
|---|---|---|
|
|
40
|
+
| `hf` | `Trainer(..., callbacks=[RunyardCallback()])` | per training; TrainingArguments and model config; `train/`, `eval/` by global step |
|
|
41
|
+
| `accelerate` | `Accelerator(log_with=[RunyardTracker(project=...)])` | per `init_trackers`; its config; `accelerator.log()` |
|
|
42
|
+
| `lightning` | `Trainer(logger=RunyardLogger(project=...))` | per Trainer call; hyperparameters; logged metrics by global step |
|
|
43
|
+
| `keras` | `model.fit(..., callbacks=[RunyardCallback(project=...)])` | per fit; fit params and parameter count; `train/`, `val/` by epoch, `batch/` with `log_every_n_batches` |
|
|
44
|
+
| `xgboost` | `xgb.train(..., callbacks=[RunyardCallback(project=...)])` | per training; booster training params; `<eval name>/<metric>` by iteration |
|
|
45
|
+
| `lightgbm` | `lgb.train(..., callbacks=[runyard_callback(project=...)])` | per training; params; `<valid name>/<metric>` by iteration |
|
|
46
|
+
| `sklearn` | `autolog(project=...)`; `autolog(disable=True)` undoes it | per outer `fit()`, named after the class; `get_params(deep=False)`; `fit_seconds`, `train/score` |
|
|
47
|
+
| `torchtune` | `metric_logger: {_component_: runyard.integrations.torchtune.RunyardLogger, project: ...}` | per recipe; resolved recipe config; the recipe's metrics by step |
|
|
48
|
+
|
|
49
|
+
On macOS, xgboost and lightgbm need `brew install libomp`, and they crash if `torch` was imported first: import them
|
|
50
|
+
before torch.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
runyard/__init__.py,sha256=nY32d2zZUl_b-eVs6HyZgjeldkN_e7SNdcbTaxGfMyE,1707
|
|
2
|
+
runyard/cli.py,sha256=4u7jNt2IWa0pNVgvIwrPeyJ9aFpMIU1J6-oAAbcAPIQ,8781
|
|
3
|
+
runyard/client.py,sha256=RoFQCCqDbFlbqk51Uokfav_XuQxepOkIYxs7p40HMCo,2884
|
|
4
|
+
runyard/core.py,sha256=c_37TaP2Q0d-TK2ZLgiRGcqrDq-DA3adYP8OsSXOyCI,18938
|
|
5
|
+
runyard/media.py,sha256=abOpejrC0kjRJuwTvPf4lQ6D_JFm2xnQE4fmyFzjv_Y,3677
|
|
6
|
+
runyard/wandb_hook.py,sha256=b83LoWcRbORnmRgA_Fr2nJzg0IkF-g8kW3bRXhRejNs,1892
|
|
7
|
+
runyard/wandb_shim.py,sha256=9FOaeWEgrt9II6aQ92iGqqLAYDFEY7W8uI1UKkE75vc,8546
|
|
8
|
+
runyard/watch.py,sha256=mWuk1BUg6pb_aZPT03kbPKujZgmVWdgDANApeEAZtbc,4979
|
|
9
|
+
runyard/integrations/__init__.py,sha256=6i6G8UWpO-eVGiU5527LBySTPPIJRGkbuLN7-UHELmw,173
|
|
10
|
+
runyard/integrations/accelerate.py,sha256=c1l0thgqJPE_gtuO05FM5dBPOwVygyo86_97rJOGD4k,1408
|
|
11
|
+
runyard/integrations/hf.py,sha256=F57zq3wwG_EcwdX4LszMnjobcp8hBjYQ67aLgHHaMiY,2214
|
|
12
|
+
runyard/integrations/keras.py,sha256=MXyLyLaer6bZ0Z2NPp32inwfDF-_Y1zNKJHYDxOvGII,1473
|
|
13
|
+
runyard/integrations/lightgbm.py,sha256=nwVevo4HDSeAkZ3CGGTjw07wHCL3R5jcfvQilmm7Rdg,1081
|
|
14
|
+
runyard/integrations/lightning.py,sha256=jV1Of4MptGams2odkvH1pVm_yAzHgIGgAgI_19lHV5I,1981
|
|
15
|
+
runyard/integrations/sklearn.py,sha256=yn_RaG63hY3Fe6ilsSnSH28ZMgr6XpLFbAdElk5Pyu0,2575
|
|
16
|
+
runyard/integrations/torchtune.py,sha256=qOzzJNCoS2H0ZBfc2a-nzrFje_w_pSgSvxD3F_HLZcc,1227
|
|
17
|
+
runyard/integrations/xgboost.py,sha256=tyQ-ive67b74ZhA7vhoSQzQczolYUTOqyEJS9WFsTyI,1736
|
|
18
|
+
runyard_wandb.pth,sha256=HTeAOC_XW2UEBkm4v86xRzWuNeC56lwiAxfghHQUrUI,122
|
|
19
|
+
runyard-0.3.0.dist-info/METADATA,sha256=sqAcgfG6z2hpvTtPfqT4-nTZYMA2yJC_fvwWgT3UsUE,2849
|
|
20
|
+
runyard-0.3.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
21
|
+
runyard-0.3.0.dist-info/entry_points.txt,sha256=acuvL_GKwvOKPeEyPMgU28Aekp5yxeqSk_i3VphoRlw,45
|
|
22
|
+
runyard-0.3.0.dist-info/RECORD,,
|
runyard_wandb.pth
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import os; os.environ.get("RUNYARD_WANDB", "") not in ("", "0") and __import__("runyard.wandb_hook").wandb_hook.install()
|