runyard 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {runyard-0.3.0 → runyard-0.4.0}/.gitignore +1 -0
- {runyard-0.3.0 → runyard-0.4.0}/PKG-INFO +1 -1
- {runyard-0.3.0 → runyard-0.4.0}/runyard/__init__.py +1 -1
- runyard-0.4.0/runyard/agent.py +114 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/cli.py +25 -0
- runyard-0.4.0/tests/test_agent.py +21 -0
- {runyard-0.3.0 → runyard-0.4.0}/README.md +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/pyproject.toml +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/client.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/core.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/__init__.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/accelerate.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/hf.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/keras.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/lightgbm.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/lightning.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/sklearn.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/torchtune.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/integrations/xgboost.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/media.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/wandb_hook.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/wandb_shim.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard/watch.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/runyard_wandb.pth +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/tests/conftest.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/tests/test_adapters.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/tests/test_integrations.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/tests/test_run.py +0 -0
- {runyard-0.3.0 → runyard-0.4.0}/tests/test_watch.py +0 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""runyard agent: posts this machine's GPUs, CPU, memory and disk to the server every few seconds, training or not,
|
|
2
|
+
so the Machines page shows who is using a GPU (any process, not only runyard runs) and the server can tell you when
|
|
3
|
+
the machine goes silent. GPUs need nvidia-ml-py, CPU/RAM/disk and process names need psutil: pip install 'runyard[system]'."""
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
import urllib.parse
|
|
9
|
+
|
|
10
|
+
from .client import request
|
|
11
|
+
|
|
12
|
+
_ARG = re.compile(r"^[\w.+/-]{1,80}$") # a script or module name; anything else may be a secret passed as an argument
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Machine:
|
|
16
|
+
def __init__(self, disk="/"):
|
|
17
|
+
self.disk = disk
|
|
18
|
+
self.nvml = self.psutil = None
|
|
19
|
+
self.gpus = []
|
|
20
|
+
try:
|
|
21
|
+
import pynvml
|
|
22
|
+
|
|
23
|
+
pynvml.nvmlInit()
|
|
24
|
+
self.gpus = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(pynvml.nvmlDeviceGetCount())]
|
|
25
|
+
self.nvml = pynvml
|
|
26
|
+
except Exception:
|
|
27
|
+
pass
|
|
28
|
+
try:
|
|
29
|
+
import psutil
|
|
30
|
+
|
|
31
|
+
psutil.cpu_percent() # the first call only starts the measurement
|
|
32
|
+
self.psutil = psutil
|
|
33
|
+
except Exception:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
def read(self):
|
|
37
|
+
"""One reading in the shape POST /api/v1/hosts/{name} takes; what this machine cannot measure is left out."""
|
|
38
|
+
v = {}
|
|
39
|
+
ps = self.psutil
|
|
40
|
+
if ps:
|
|
41
|
+
v["cpu"] = ps.cpu_percent()
|
|
42
|
+
v["cores"] = ps.cpu_count()
|
|
43
|
+
vm = ps.virtual_memory()
|
|
44
|
+
v["ram_gb"], v["ram_total_gb"] = (vm.total - vm.available) / 1e9, vm.total / 1e9
|
|
45
|
+
du = ps.disk_usage(self.disk)
|
|
46
|
+
v["disk_gb"], v["disk_total_gb"] = du.used / 1e9, du.total / 1e9
|
|
47
|
+
try:
|
|
48
|
+
v["load"] = os.getloadavg()[0]
|
|
49
|
+
except (AttributeError, OSError): # Windows
|
|
50
|
+
pass
|
|
51
|
+
v["gpus"] = [self._gpu(i, h) for i, h in enumerate(self.gpus)]
|
|
52
|
+
return v
|
|
53
|
+
|
|
54
|
+
def _gpu(self, i, h):
|
|
55
|
+
n = self.nvml
|
|
56
|
+
g = {"i": i}
|
|
57
|
+
reads = { # each on its own: consumer cards refuse some (power limit, processes)
|
|
58
|
+
"name": lambda: _text(n.nvmlDeviceGetName(h)),
|
|
59
|
+
"util": lambda: n.nvmlDeviceGetUtilizationRates(h).gpu,
|
|
60
|
+
"mem_gb": lambda: n.nvmlDeviceGetMemoryInfo(h).used / 1e9,
|
|
61
|
+
"mem_total_gb": lambda: n.nvmlDeviceGetMemoryInfo(h).total / 1e9,
|
|
62
|
+
"temp_c": lambda: n.nvmlDeviceGetTemperature(h, 0),
|
|
63
|
+
"power_w": lambda: n.nvmlDeviceGetPowerUsage(h) / 1000,
|
|
64
|
+
"power_max_w": lambda: n.nvmlDeviceGetEnforcedPowerLimit(h) / 1000,
|
|
65
|
+
"procs": lambda: [self._proc(p) for p in n.nvmlDeviceGetComputeRunningProcesses(h)],
|
|
66
|
+
}
|
|
67
|
+
for k, f in reads.items():
|
|
68
|
+
try:
|
|
69
|
+
g[k] = f()
|
|
70
|
+
except Exception:
|
|
71
|
+
pass
|
|
72
|
+
return g
|
|
73
|
+
|
|
74
|
+
def _proc(self, p):
|
|
75
|
+
out = {"pid": p.pid, "name": "", "mem_gb": (p.usedGpuMemory or 0) / 1e9}
|
|
76
|
+
try:
|
|
77
|
+
proc = self.psutil.Process(p.pid)
|
|
78
|
+
argv = proc.cmdline()
|
|
79
|
+
prog = os.path.basename(argv[0]) if argv else proc.name()
|
|
80
|
+
script = next((a for a in argv[1:] if not a.startswith("-")), "")
|
|
81
|
+
out["name"] = f"{prog} {os.path.basename(script)}" if _ARG.match(script) else prog
|
|
82
|
+
out["user"] = proc.username()
|
|
83
|
+
except Exception: # no psutil, or the process just ended
|
|
84
|
+
pass
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _text(s):
|
|
89
|
+
return s.decode() if isinstance(s, bytes) else s
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def agent(server, key, name, interval=10, once=False, disk="/"):
|
|
93
|
+
"""Posts a reading every interval seconds until stopped; a failure is reported once and retried."""
|
|
94
|
+
m = Machine(disk)
|
|
95
|
+
url = f"{server}/api/v1/hosts/{urllib.parse.quote(name)}"
|
|
96
|
+
print(f"runyard agent: {name} -> {server}, every {interval:g}s ({len(m.gpus)} GPUs"
|
|
97
|
+
f"{'' if m.psutil else ', no psutil: no CPU, RAM or disk'})", file=sys.stderr, flush=True)
|
|
98
|
+
failing = False
|
|
99
|
+
while True:
|
|
100
|
+
start = time.monotonic()
|
|
101
|
+
try:
|
|
102
|
+
request("POST", url, key, m.read())
|
|
103
|
+
if failing:
|
|
104
|
+
print("runyard agent: sending again", file=sys.stderr, flush=True)
|
|
105
|
+
failing = False
|
|
106
|
+
except Exception as e:
|
|
107
|
+
if once:
|
|
108
|
+
raise
|
|
109
|
+
if not failing:
|
|
110
|
+
print(f"runyard agent: {e}; retrying", file=sys.stderr, flush=True)
|
|
111
|
+
failing = True
|
|
112
|
+
if once:
|
|
113
|
+
return
|
|
114
|
+
time.sleep(max(0.0, interval - (time.monotonic() - start)))
|
|
@@ -9,11 +9,15 @@
|
|
|
9
9
|
runyard watch DIR [--project P] [--once] [--interval 10]
|
|
10
10
|
sends TensorBoard event files under DIR, one run per directory, until Ctrl-C;
|
|
11
11
|
--once reads them once and exits (needs the tensorboard package)
|
|
12
|
+
runyard agent [--name NAME] [--interval 10]
|
|
13
|
+
sends this machine's GPU, CPU, memory and disk to the server until stopped,
|
|
14
|
+
for the Machines page (run it as a service, see docs/server.md)
|
|
12
15
|
"""
|
|
13
16
|
import argparse
|
|
14
17
|
import json
|
|
15
18
|
import os
|
|
16
19
|
import signal
|
|
20
|
+
import socket
|
|
17
21
|
import sys
|
|
18
22
|
import time
|
|
19
23
|
import urllib.parse
|
|
@@ -164,6 +168,20 @@ def cmd_watch(a):
|
|
|
164
168
|
watch(a.dir, a.project, a.once, a.interval)
|
|
165
169
|
|
|
166
170
|
|
|
171
|
+
def cmd_agent(a):
|
|
172
|
+
"""Sends this machine's GPU, CPU, memory and disk to the server every few seconds, until stopped."""
|
|
173
|
+
if not key():
|
|
174
|
+
print("runyard: agent needs RUNYARD_API_KEY, a key from `runyard-server key create`", file=sys.stderr)
|
|
175
|
+
return 1
|
|
176
|
+
from .agent import agent
|
|
177
|
+
|
|
178
|
+
signal.signal(signal.SIGTERM, signal.default_int_handler) # systemd and launchd stop it like Ctrl-C
|
|
179
|
+
try:
|
|
180
|
+
agent(server(), key(), a.name, a.interval, a.once, a.disk)
|
|
181
|
+
except KeyboardInterrupt:
|
|
182
|
+
pass
|
|
183
|
+
|
|
184
|
+
|
|
167
185
|
def main(argv=None):
|
|
168
186
|
p = argparse.ArgumentParser(prog="runyard", description="Reads runs from the server in RUNYARD_SERVER; watch sends to it.")
|
|
169
187
|
p.add_argument("--version", action="version", version=__version__)
|
|
@@ -207,6 +225,13 @@ def main(argv=None):
|
|
|
207
225
|
watch.add_argument("--once", action="store_true", help="read everything once, finish the runs and exit (imports)")
|
|
208
226
|
watch.add_argument("--interval", type=float, default=10, help="seconds between polls (default 10)")
|
|
209
227
|
watch.set_defaults(fn=cmd_watch)
|
|
228
|
+
|
|
229
|
+
ag = sub.add_parser("agent", help=cmd_agent.__doc__)
|
|
230
|
+
ag.add_argument("--name", default=socket.gethostname(), help="this machine's name (default: its hostname, as runs record it)")
|
|
231
|
+
ag.add_argument("--interval", type=float, default=10, help="seconds between readings (default 10)")
|
|
232
|
+
ag.add_argument("--disk", default="/", help="the filesystem whose usage to report (default /)")
|
|
233
|
+
ag.add_argument("--once", action="store_true", help="send one reading and exit")
|
|
234
|
+
ag.set_defaults(fn=cmd_agent)
|
|
210
235
|
a = p.parse_args(argv)
|
|
211
236
|
try:
|
|
212
237
|
return a.fn(a) or 0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""runyard agent: a reading lands on the Machines endpoint; GPU processes are named without their arguments."""
|
|
2
|
+
from types import SimpleNamespace
|
|
3
|
+
|
|
4
|
+
from conftest import get
|
|
5
|
+
from runyard.agent import Machine
|
|
6
|
+
from runyard.cli import main
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_agent_once(server):
|
|
10
|
+
assert main(["agent", "--name", "box-1", "--once"]) == 0
|
|
11
|
+
hosts = get(server + "/api/v1/hosts")
|
|
12
|
+
assert [h["name"] for h in hosts] == ["box-1"]
|
|
13
|
+
assert isinstance(hosts[0]["stats"].get("gpus", []), list)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_proc_name_keeps_secrets_out():
|
|
17
|
+
m = Machine()
|
|
18
|
+
argv = {1: ["/usr/bin/python3", "-u", "train.py", "--lr", "3e-4"], 2: ["curl", "-H", "Authorization: Bearer xyz"]}
|
|
19
|
+
m.psutil = SimpleNamespace(Process=lambda pid: SimpleNamespace(cmdline=lambda: argv[pid], name=lambda: "?", username=lambda: "ege"))
|
|
20
|
+
assert m._proc(SimpleNamespace(pid=1, usedGpuMemory=2e9)) == {"pid": 1, "name": "python3 train.py", "user": "ege", "mem_gb": 2.0}
|
|
21
|
+
assert m._proc(SimpleNamespace(pid=2, usedGpuMemory=None))["name"] == "curl"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|