runyard 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
runyard/watch.py ADDED
@@ -0,0 +1,102 @@
1
+ """`runyard watch DIR`: sends TensorBoard event files to runyard, for tools that write nothing else (kohya sd-scripts,
2
+ DeepSpeed, diffusers --report_to tensorboard, torch SummaryWriter). Needs the tensorboard package.
3
+
4
+ Each directory holding events.out.tfevents.* files is one run, named by its path under DIR. Scalars, images, text
5
+ and hparams are read; other summaries (histograms, audio, graphs) are skipped.
6
+ """
7
+ import json
8
+ import os
9
+ import re
10
+ import socket
11
+ import time
12
+ import uuid
13
+ from datetime import datetime, timezone
14
+ from pathlib import Path
15
+
16
+ from tensorboard.backend.event_processing.event_file_loader import EventFileLoader
17
+ from tensorboard.plugins.hparams import plugin_data_pb2
18
+ from tensorboard.util import tensor_util
19
+
20
+ from .core import Run, _safe, warn
21
+
22
+
23
+ def run_dir(f, root):
24
+ """An event file's run directory: its own, or the parent of torch's add_hparams subdirectory (<logdir>/<time>)."""
25
+ d = f.parent
26
+ return d.parent if d != root and re.fullmatch(r"\d{9,}\.\d+", d.name) else d
27
+
28
+
29
+ def start(d, root, project, live):
30
+ name = d.relative_to(root).as_posix() if d != root else root.name
31
+ # The same directory on the same machine is always the same run, so a restarted watcher continues it.
32
+ rid = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{socket.gethostname()}:{d}"))
33
+ # shortcut: started_at is when the watcher first saw the directory, not the first event's time; upgrade by letting
34
+ # Run take started_at if imports of old trainings sort wrongly in the run list.
35
+ run = Run(project, name, id=rid, capture_output=False, system=False)
36
+ if live:
37
+ run._patch({"status": "running"}) # an earlier `watch --once` may have finished it
38
+ print(f"{name}: {run.url}", flush=True)
39
+ return run
40
+
41
+
42
+ def save(run, kind, key, ev, i, data, ext, meta):
43
+ # A fixed id per sample: the server ignores the ones a restarted watcher reads again.
44
+ mid = uuid.uuid5(uuid.UUID(run.id), f"{key}/{ev.step}/{i}/{ev.wall_time}")
45
+ path = run._dir / f"{mid}{ext}"
46
+ path.parent.mkdir(parents=True, exist_ok=True)
47
+ path.write_bytes(data)
48
+ run._media(key, kind, ev.step, path, meta)
49
+
50
+
51
+ @_safe
52
+ def send(run, ev, plugins):
53
+ """Sends one event's summaries. plugins maps tag to plugin name, which TF2 writers put on a tag's first value only."""
54
+ scalars = {}
55
+ for v in ev.summary.value:
56
+ plugin = plugins.setdefault(v.tag, v.metadata.plugin_data.plugin_name)
57
+ if plugin == "scalars":
58
+ scalars[v.tag] = tensor_util.make_ndarray(v.tensor).item()
59
+ elif plugin == "images":
60
+ imgs = v.tensor.string_val[2:] # width, height, then the encoded images
61
+ for i, b in enumerate(imgs):
62
+ ext = ".png" if b[:4] == b"\x89PNG" else ".gif" if b[:4] == b"GIF8" else ".jpg"
63
+ meta = {"caption": None, "index": i} if len(imgs) > 1 else {"caption": None}
64
+ save(run, "image", v.tag, ev, i, b, ext, meta)
65
+ elif plugin == "text":
66
+ rows = [{"text": s.decode("utf-8", "replace")} for s in v.tensor.string_val]
67
+ key = v.tag.removesuffix("/text_summary") # torch's add_text appends it
68
+ save(run, "text", key, ev, 0, json.dumps(rows).encode(), ".json", {"rows": len(rows)})
69
+ elif plugin == "hparams":
70
+ hp = plugin_data_pb2.HParamsPluginData.FromString(v.metadata.plugin_data.content).session_start_info.hparams
71
+ if hp:
72
+ run.config.update({k: getattr(x, x.WhichOneof("kind")) for k, x in hp.items() if x.WhichOneof("kind")})
73
+ else:
74
+ warn(f"skipped {plugin or 'untyped'} summaries: watch reads scalars, images, text and hparams", once=True)
75
+ if scalars:
76
+ run.log(scalars, step=ev.step, ts=datetime.fromtimestamp(ev.wall_time, timezone.utc).isoformat())
77
+
78
+
79
+ # shortcut: a run stays running while the watcher runs, as event files say nothing about a training's end; upgrade to
80
+ # finishing a run after some idle time if long-lived watchers over many finished trainings become common.
81
+ def watch(root, project, once=False, interval=10):
82
+ """Reads every event file under root, then (unless once) polls for more until Ctrl-C; finishes the runs."""
83
+ root = Path(root).resolve()
84
+ project = project or os.environ.get("RUNYARD_PROJECT") or root.name
85
+ runs, loaders, plugins = {}, {}, {}
86
+ try:
87
+ while True:
88
+ for f in sorted(root.rglob("events.out.tfevents.*")):
89
+ d = run_dir(f, root)
90
+ if d not in runs:
91
+ runs[d] = start(d, root, project, live=not once)
92
+ if f not in loaders:
93
+ loaders[f] = EventFileLoader(str(f)) # keeps its place in the file, so polls read only new events
94
+ for ev in loaders[f].Load():
95
+ send(runs[d], ev, plugins.setdefault(d, {}))
96
+ if once:
97
+ break
98
+ time.sleep(interval)
99
+ except KeyboardInterrupt:
100
+ pass
101
+ for run in runs.values():
102
+ run.finish()
@@ -0,0 +1,50 @@
1
+ Metadata-Version: 2.5
2
+ Name: runyard
3
+ Version: 0.3.0
4
+ Summary: Python client for runyard, a lightweight self-hosted training tracker
5
+ Project-URL: Homepage, https://github.com/cobanov/runyard
6
+ License-Expression: Apache-2.0
7
+ Requires-Python: >=3.9
8
+ Provides-Extra: dev
9
+ Requires-Dist: numpy; extra == 'dev'
10
+ Requires-Dist: pillow; extra == 'dev'
11
+ Requires-Dist: pytest; extra == 'dev'
12
+ Provides-Extra: images
13
+ Requires-Dist: numpy; extra == 'images'
14
+ Requires-Dist: pillow; extra == 'images'
15
+ Provides-Extra: system
16
+ Requires-Dist: nvidia-ml-py; extra == 'system'
17
+ Requires-Dist: psutil; extra == 'system'
18
+ Provides-Extra: tensorboard
19
+ Requires-Dist: tensorboard; extra == 'tensorboard'
20
+ Provides-Extra: video
21
+ Requires-Dist: imageio; extra == 'video'
22
+ Requires-Dist: imageio-ffmpeg; extra == 'video'
23
+ Description-Content-Type: text/markdown
24
+
25
+ # runyard (Python client)
26
+
27
+ See the [main README](https://github.com/cobanov/runyard).
28
+
29
+ Optional extras: `images` (numpy, Pillow), `system` (GPU, CPU and RAM usage), `video` (imageio-ffmpeg: frames
30
+ become an H.264 mp4 instead of a GIF) and `tensorboard` (for `runyard watch DIR`, which sends TensorBoard event
31
+ files: one run per directory, `--once` to import and exit). `runyard --help` lists the commands.
32
+
33
+ ## Framework adapters
34
+
35
+ `runyard.integrations.<module>` imports its framework, which you install yourself. Keyword arguments other than
36
+ `project` and `name` go to `runyard.init` (`tags`, `notes`, `system`, ...).
37
+
38
+ | Module | One line | Run, config, metrics |
39
+ |---|---|---|
40
+ | `hf` | `Trainer(..., callbacks=[RunyardCallback()])` | per training; TrainingArguments and model config; `train/`, `eval/` by global step |
41
+ | `accelerate` | `Accelerator(log_with=[RunyardTracker(project=...)])` | per `init_trackers`; its config; `accelerator.log()` |
42
+ | `lightning` | `Trainer(logger=RunyardLogger(project=...))` | per Trainer call; hyperparameters; logged metrics by global step |
43
+ | `keras` | `model.fit(..., callbacks=[RunyardCallback(project=...)])` | per fit; fit params and parameter count; `train/`, `val/` by epoch, `batch/` with `log_every_n_batches` |
44
+ | `xgboost` | `xgb.train(..., callbacks=[RunyardCallback(project=...)])` | per training; booster training params; `<eval name>/<metric>` by iteration |
45
+ | `lightgbm` | `lgb.train(..., callbacks=[runyard_callback(project=...)])` | per training; params; `<valid name>/<metric>` by iteration |
46
+ | `sklearn` | `autolog(project=...)`; `autolog(disable=True)` undoes it | per outer `fit()`, named after the class; `get_params(deep=False)`; `fit_seconds`, `train/score` |
47
+ | `torchtune` | `metric_logger: {_component_: runyard.integrations.torchtune.RunyardLogger, project: ...}` | per recipe; resolved recipe config; the recipe's metrics by step |
48
+
49
+ On macOS, xgboost and lightgbm need `brew install libomp`, and they crash if `torch` was imported first: import them
50
+ before torch.
@@ -0,0 +1,22 @@
1
+ runyard/__init__.py,sha256=nY32d2zZUl_b-eVs6HyZgjeldkN_e7SNdcbTaxGfMyE,1707
2
+ runyard/cli.py,sha256=4u7jNt2IWa0pNVgvIwrPeyJ9aFpMIU1J6-oAAbcAPIQ,8781
3
+ runyard/client.py,sha256=RoFQCCqDbFlbqk51Uokfav_XuQxepOkIYxs7p40HMCo,2884
4
+ runyard/core.py,sha256=c_37TaP2Q0d-TK2ZLgiRGcqrDq-DA3adYP8OsSXOyCI,18938
5
+ runyard/media.py,sha256=abOpejrC0kjRJuwTvPf4lQ6D_JFm2xnQE4fmyFzjv_Y,3677
6
+ runyard/wandb_hook.py,sha256=b83LoWcRbORnmRgA_Fr2nJzg0IkF-g8kW3bRXhRejNs,1892
7
+ runyard/wandb_shim.py,sha256=9FOaeWEgrt9II6aQ92iGqqLAYDFEY7W8uI1UKkE75vc,8546
8
+ runyard/watch.py,sha256=mWuk1BUg6pb_aZPT03kbPKujZgmVWdgDANApeEAZtbc,4979
9
+ runyard/integrations/__init__.py,sha256=6i6G8UWpO-eVGiU5527LBySTPPIJRGkbuLN7-UHELmw,173
10
+ runyard/integrations/accelerate.py,sha256=c1l0thgqJPE_gtuO05FM5dBPOwVygyo86_97rJOGD4k,1408
11
+ runyard/integrations/hf.py,sha256=F57zq3wwG_EcwdX4LszMnjobcp8hBjYQ67aLgHHaMiY,2214
12
+ runyard/integrations/keras.py,sha256=MXyLyLaer6bZ0Z2NPp32inwfDF-_Y1zNKJHYDxOvGII,1473
13
+ runyard/integrations/lightgbm.py,sha256=nwVevo4HDSeAkZ3CGGTjw07wHCL3R5jcfvQilmm7Rdg,1081
14
+ runyard/integrations/lightning.py,sha256=jV1Of4MptGams2odkvH1pVm_yAzHgIGgAgI_19lHV5I,1981
15
+ runyard/integrations/sklearn.py,sha256=yn_RaG63hY3Fe6ilsSnSH28ZMgr6XpLFbAdElk5Pyu0,2575
16
+ runyard/integrations/torchtune.py,sha256=qOzzJNCoS2H0ZBfc2a-nzrFje_w_pSgSvxD3F_HLZcc,1227
17
+ runyard/integrations/xgboost.py,sha256=tyQ-ive67b74ZhA7vhoSQzQczolYUTOqyEJS9WFsTyI,1736
18
+ runyard_wandb.pth,sha256=HTeAOC_XW2UEBkm4v86xRzWuNeC56lwiAxfghHQUrUI,122
19
+ runyard-0.3.0.dist-info/METADATA,sha256=sqAcgfG6z2hpvTtPfqT4-nTZYMA2yJC_fvwWgT3UsUE,2849
20
+ runyard-0.3.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
21
+ runyard-0.3.0.dist-info/entry_points.txt,sha256=acuvL_GKwvOKPeEyPMgU28Aekp5yxeqSk_i3VphoRlw,45
22
+ runyard-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ runyard = runyard.cli:main
runyard_wandb.pth ADDED
@@ -0,0 +1 @@
1
+ import os; os.environ.get("RUNYARD_WANDB", "") not in ("", "0") and __import__("runyard.wandb_hook").wandb_hook.install()