runyard 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {runyard-0.3.0 → runyard-0.4.0}/.gitignore +1 -0
  2. {runyard-0.3.0 → runyard-0.4.0}/PKG-INFO +1 -1
  3. {runyard-0.3.0 → runyard-0.4.0}/runyard/__init__.py +1 -1
  4. runyard-0.4.0/runyard/agent.py +114 -0
  5. {runyard-0.3.0 → runyard-0.4.0}/runyard/cli.py +25 -0
  6. runyard-0.4.0/tests/test_agent.py +21 -0
  7. {runyard-0.3.0 → runyard-0.4.0}/README.md +0 -0
  8. {runyard-0.3.0 → runyard-0.4.0}/pyproject.toml +0 -0
  9. {runyard-0.3.0 → runyard-0.4.0}/runyard/client.py +0 -0
  10. {runyard-0.3.0 → runyard-0.4.0}/runyard/core.py +0 -0
  11. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/__init__.py +0 -0
  12. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/accelerate.py +0 -0
  13. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/hf.py +0 -0
  14. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/keras.py +0 -0
  15. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/lightgbm.py +0 -0
  16. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/lightning.py +0 -0
  17. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/sklearn.py +0 -0
  18. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/torchtune.py +0 -0
  19. {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/xgboost.py +0 -0
  20. {runyard-0.3.0 → runyard-0.4.0}/runyard/media.py +0 -0
  21. {runyard-0.3.0 → runyard-0.4.0}/runyard/wandb_hook.py +0 -0
  22. {runyard-0.3.0 → runyard-0.4.0}/runyard/wandb_shim.py +0 -0
  23. {runyard-0.3.0 → runyard-0.4.0}/runyard/watch.py +0 -0
  24. {runyard-0.3.0 → runyard-0.4.0}/runyard_wandb.pth +0 -0
  25. {runyard-0.3.0 → runyard-0.4.0}/tests/conftest.py +0 -0
  26. {runyard-0.3.0 → runyard-0.4.0}/tests/test_adapters.py +0 -0
  27. {runyard-0.3.0 → runyard-0.4.0}/tests/test_integrations.py +0 -0
  28. {runyard-0.3.0 → runyard-0.4.0}/tests/test_run.py +0 -0
  29. {runyard-0.3.0 → runyard-0.4.0}/tests/test_watch.py +0 -0
@@ -19,3 +19,4 @@ build/
19
19
  # private environment notes, see plan.md §16
20
20
  /ops-notes.md
21
21
  /server/server
22
+ .wrangler/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: runyard
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Python client for runyard, a lightweight self-hosted training tracker
5
5
  Project-URL: Homepage, https://github.com/cobanov/runyard
6
6
  License-Expression: Apache-2.0
@@ -10,7 +10,7 @@ Settings come from RUNYARD_SERVER, RUNYARD_API_KEY and RUNYARD_PROJECT.
10
10
  """
11
11
  from .core import Run, sync, warn
12
12
 
13
- __version__ = "0.3.0"
13
+ __version__ = "0.4.0"
14
14
 
15
15
  run = None # the run the module-level shortcuts act on: the last one init() made
16
16
 
@@ -0,0 +1,114 @@
1
+ """runyard agent: posts this machine's GPUs, CPU, memory and disk to the server every few seconds, training or not,
2
+ so the Machines page shows who is using a GPU (any process, not only runyard runs) and the server can tell you when
3
+ the machine goes silent. GPUs need nvidia-ml-py, CPU/RAM/disk and process names need psutil: pip install 'runyard[system]'."""
4
+ import os
5
+ import re
6
+ import sys
7
+ import time
8
+ import urllib.parse
9
+
10
+ from .client import request
11
+
12
+ _ARG = re.compile(r"^[\w.+/-]{1,80}$") # a script or module name; anything else may be a secret passed as an argument
13
+
14
+
15
+ class Machine:
16
+ def __init__(self, disk="/"):
17
+ self.disk = disk
18
+ self.nvml = self.psutil = None
19
+ self.gpus = []
20
+ try:
21
+ import pynvml
22
+
23
+ pynvml.nvmlInit()
24
+ self.gpus = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(pynvml.nvmlDeviceGetCount())]
25
+ self.nvml = pynvml
26
+ except Exception:
27
+ pass
28
+ try:
29
+ import psutil
30
+
31
+ psutil.cpu_percent() # the first call only starts the measurement
32
+ self.psutil = psutil
33
+ except Exception:
34
+ pass
35
+
36
+ def read(self):
37
+ """One reading in the shape POST /api/v1/hosts/{name} takes; what this machine cannot measure is left out."""
38
+ v = {}
39
+ ps = self.psutil
40
+ if ps:
41
+ v["cpu"] = ps.cpu_percent()
42
+ v["cores"] = ps.cpu_count()
43
+ vm = ps.virtual_memory()
44
+ v["ram_gb"], v["ram_total_gb"] = (vm.total - vm.available) / 1e9, vm.total / 1e9
45
+ du = ps.disk_usage(self.disk)
46
+ v["disk_gb"], v["disk_total_gb"] = du.used / 1e9, du.total / 1e9
47
+ try:
48
+ v["load"] = os.getloadavg()[0]
49
+ except (AttributeError, OSError): # Windows
50
+ pass
51
+ v["gpus"] = [self._gpu(i, h) for i, h in enumerate(self.gpus)]
52
+ return v
53
+
54
+ def _gpu(self, i, h):
55
+ n = self.nvml
56
+ g = {"i": i}
57
+ reads = { # each on its own: consumer cards refuse some (power limit, processes)
58
+ "name": lambda: _text(n.nvmlDeviceGetName(h)),
59
+ "util": lambda: n.nvmlDeviceGetUtilizationRates(h).gpu,
60
+ "mem_gb": lambda: n.nvmlDeviceGetMemoryInfo(h).used / 1e9,
61
+ "mem_total_gb": lambda: n.nvmlDeviceGetMemoryInfo(h).total / 1e9,
62
+ "temp_c": lambda: n.nvmlDeviceGetTemperature(h, 0),
63
+ "power_w": lambda: n.nvmlDeviceGetPowerUsage(h) / 1000,
64
+ "power_max_w": lambda: n.nvmlDeviceGetEnforcedPowerLimit(h) / 1000,
65
+ "procs": lambda: [self._proc(p) for p in n.nvmlDeviceGetComputeRunningProcesses(h)],
66
+ }
67
+ for k, f in reads.items():
68
+ try:
69
+ g[k] = f()
70
+ except Exception:
71
+ pass
72
+ return g
73
+
74
+ def _proc(self, p):
75
+ out = {"pid": p.pid, "name": "", "mem_gb": (p.usedGpuMemory or 0) / 1e9}
76
+ try:
77
+ proc = self.psutil.Process(p.pid)
78
+ argv = proc.cmdline()
79
+ prog = os.path.basename(argv[0]) if argv else proc.name()
80
+ script = next((a for a in argv[1:] if not a.startswith("-")), "")
81
+ out["name"] = f"{prog} {os.path.basename(script)}" if _ARG.match(script) else prog
82
+ out["user"] = proc.username()
83
+ except Exception: # no psutil, or the process just ended
84
+ pass
85
+ return out
86
+
87
+
88
+ def _text(s):
89
+ return s.decode() if isinstance(s, bytes) else s
90
+
91
+
92
+ def agent(server, key, name, interval=10, once=False, disk="/"):
93
+ """Posts a reading every interval seconds until stopped; a failure is reported once and retried."""
94
+ m = Machine(disk)
95
+ url = f"{server}/api/v1/hosts/{urllib.parse.quote(name)}"
96
+ print(f"runyard agent: {name} -> {server}, every {interval:g}s ({len(m.gpus)} GPUs"
97
+ f"{'' if m.psutil else ', no psutil: no CPU, RAM or disk'})", file=sys.stderr, flush=True)
98
+ failing = False
99
+ while True:
100
+ start = time.monotonic()
101
+ try:
102
+ request("POST", url, key, m.read())
103
+ if failing:
104
+ print("runyard agent: sending again", file=sys.stderr, flush=True)
105
+ failing = False
106
+ except Exception as e:
107
+ if once:
108
+ raise
109
+ if not failing:
110
+ print(f"runyard agent: {e}; retrying", file=sys.stderr, flush=True)
111
+ failing = True
112
+ if once:
113
+ return
114
+ time.sleep(max(0.0, interval - (time.monotonic() - start)))
@@ -9,11 +9,15 @@
9
9
  runyard watch DIR [--project P] [--once] [--interval 10]
10
10
  sends TensorBoard event files under DIR, one run per directory, until Ctrl-C;
11
11
  --once reads them once and exits (needs the tensorboard package)
12
+ runyard agent [--name NAME] [--interval 10]
13
+ sends this machine's GPU, CPU, memory and disk to the server until stopped,
14
+ for the Machines page (run it as a service, see docs/server.md)
12
15
  """
13
16
  import argparse
14
17
  import json
15
18
  import os
16
19
  import signal
20
+ import socket
17
21
  import sys
18
22
  import time
19
23
  import urllib.parse
@@ -164,6 +168,20 @@ def cmd_watch(a):
164
168
  watch(a.dir, a.project, a.once, a.interval)
165
169
 
166
170
 
171
+ def cmd_agent(a):
172
+ """Sends this machine's GPU, CPU, memory and disk to the server every few seconds, until stopped."""
173
+ if not key():
174
+ print("runyard: agent needs RUNYARD_API_KEY, a key from `runyard-server key create`", file=sys.stderr)
175
+ return 1
176
+ from .agent import agent
177
+
178
+ signal.signal(signal.SIGTERM, signal.default_int_handler) # systemd and launchd stop it like Ctrl-C
179
+ try:
180
+ agent(server(), key(), a.name, a.interval, a.once, a.disk)
181
+ except KeyboardInterrupt:
182
+ pass
183
+
184
+
167
185
  def main(argv=None):
168
186
  p = argparse.ArgumentParser(prog="runyard", description="Reads runs from the server in RUNYARD_SERVER; watch sends to it.")
169
187
  p.add_argument("--version", action="version", version=__version__)
@@ -207,6 +225,13 @@ def main(argv=None):
207
225
  watch.add_argument("--once", action="store_true", help="read everything once, finish the runs and exit (imports)")
208
226
  watch.add_argument("--interval", type=float, default=10, help="seconds between polls (default 10)")
209
227
  watch.set_defaults(fn=cmd_watch)
228
+
229
+ ag = sub.add_parser("agent", help=cmd_agent.__doc__)
230
+ ag.add_argument("--name", default=socket.gethostname(), help="this machine's name (default: its hostname, as runs record it)")
231
+ ag.add_argument("--interval", type=float, default=10, help="seconds between readings (default 10)")
232
+ ag.add_argument("--disk", default="/", help="the filesystem whose usage to report (default /)")
233
+ ag.add_argument("--once", action="store_true", help="send one reading and exit")
234
+ ag.set_defaults(fn=cmd_agent)
210
235
  a = p.parse_args(argv)
211
236
  try:
212
237
  return a.fn(a) or 0
@@ -0,0 +1,21 @@
1
+ """runyard agent: a reading lands on the Machines endpoint; GPU processes are named without their arguments."""
2
+ from types import SimpleNamespace
3
+
4
+ from conftest import get
5
+ from runyard.agent import Machine
6
+ from runyard.cli import main
7
+
8
+
9
+ def test_agent_once(server):
10
+ assert main(["agent", "--name", "box-1", "--once"]) == 0
11
+ hosts = get(server + "/api/v1/hosts")
12
+ assert [h["name"] for h in hosts] == ["box-1"]
13
+ assert isinstance(hosts[0]["stats"].get("gpus", []), list)
14
+
15
+
16
+ def test_proc_name_keeps_secrets_out():
17
+ m = Machine()
18
+ argv = {1: ["/usr/bin/python3", "-u", "train.py", "--lr", "3e-4"], 2: ["curl", "-H", "Authorization: Bearer xyz"]}
19
+ m.psutil = SimpleNamespace(Process=lambda pid: SimpleNamespace(cmdline=lambda: argv[pid], name=lambda: "?", username=lambda: "ege"))
20
+ assert m._proc(SimpleNamespace(pid=1, usedGpuMemory=2e9)) == {"pid": 1, "name": "python3 train.py", "user": "ege", "mem_gb": 2.0}
21
+ assert m._proc(SimpleNamespace(pid=2, usedGpuMemory=None))["name"] == "curl"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes