nodeocc 1.0.25__tar.gz → 1.0.27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nodeocc-1.0.25 → nodeocc-1.0.27}/PKG-INFO +5 -1
- {nodeocc-1.0.25 → nodeocc-1.0.27}/README.md +4 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/pyproject.toml +1 -1
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/controller/controller.py +36 -50
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/PKG-INFO +5 -1
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/SOURCES.txt +1 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/readers/slurmreader.py +21 -7
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/__init__.py +7 -1
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/curses_multiwindow.py +118 -9
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/slurm_list.py +1 -1
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/slurm_viz.py +21 -6
- nodeocc-1.0.27/src/view/transport.py +46 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/setup.cfg +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/setup.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/__init__.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/model/infrastructure.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/model/job.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/dependency_links.txt +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/entry_points.txt +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/requires.txt +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/top_level.txt +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/readers/compute_wait_time.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/styles.py +0 -0
- {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nodeocc
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.27
|
|
4
4
|
Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
|
|
5
5
|
Author: Matteo Boschini
|
|
6
6
|
Maintainer: Lorenzo Bonicelli, Aniello Panariello
|
|
@@ -31,4 +31,8 @@ Contents:
|
|
|
31
31
|
|
|
32
32
|
Run it with: `python controller/controller.py`
|
|
33
33
|
|
|
34
|
+
Press `D` in the interactive view to open or close the debug screen. It shows
|
|
35
|
+
the local role, master PID and port, snapshot status, and local package
|
|
36
|
+
versions. Use the up and down keys to scroll.
|
|
37
|
+
|
|
34
38
|
Requirements: pandas, pycurses.
|
|
@@ -13,4 +13,8 @@ Contents:
|
|
|
13
13
|
|
|
14
14
|
Run it with: `python controller/controller.py`
|
|
15
15
|
|
|
16
|
+
Press `D` in the interactive view to open or close the debug screen. It shows
|
|
17
|
+
the local role, master PID and port, snapshot status, and local package
|
|
18
|
+
versions. Use the up and down keys to scroll.
|
|
19
|
+
|
|
16
20
|
Requirements: pandas, pycurses.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "nodeocc"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.27"
|
|
8
8
|
authors = [{ name = "Matteo Boschini" }]
|
|
9
9
|
maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
|
|
10
10
|
description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"
|
|
@@ -24,43 +24,38 @@ import json
|
|
|
24
24
|
import argparse
|
|
25
25
|
import traceback
|
|
26
26
|
import socket
|
|
27
|
+
import re
|
|
28
|
+
from urllib.request import urlopen
|
|
27
29
|
from model.infrastructure import Infrastructure
|
|
28
30
|
from model.job import Job
|
|
29
31
|
import asyncio
|
|
30
32
|
import importlib.metadata
|
|
33
|
+
from view.transport import send as send_datagram, receive as receive_datagram
|
|
31
34
|
|
|
32
35
|
program_name = 'nodeocc'
|
|
33
36
|
version_number = importlib.metadata.version(program_name)
|
|
34
|
-
updated = False
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
updated = True
|
|
58
|
-
time.sleep(3)
|
|
59
|
-
print(END, end='')
|
|
60
|
-
except Exception as e:
|
|
61
|
-
print(f"{BBLUE}Could not get newest version from pip, check if {program_name} is up to date{END}")
|
|
62
|
-
newest_version = '?.?.?'
|
|
63
|
-
time.sleep(5)
|
|
38
|
+
|
|
39
|
+
def _version_tuple(v):
|
|
40
|
+
return tuple(int(x) for x in re.findall(r'\d+', str(v)))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def check_latest_version(cur_version, timeout_sec=1.2):
|
|
44
|
+
"""
|
|
45
|
+
Fast, non-blocking-ish version check:
|
|
46
|
+
- short timeout
|
|
47
|
+
- no startup sleeps/pauses
|
|
48
|
+
- graceful fallback to current version on errors
|
|
49
|
+
"""
|
|
50
|
+
try:
|
|
51
|
+
url = f"https://pypi.python.org/pypi/{program_name}/json"
|
|
52
|
+
with urlopen(url, timeout=timeout_sec) as response:
|
|
53
|
+
payload = json.loads(response.read().decode('utf-8'))
|
|
54
|
+
newest = payload['info']['version']
|
|
55
|
+
is_updated = _version_tuple(cur_version) >= _version_tuple(newest)
|
|
56
|
+
return newest, is_updated
|
|
57
|
+
except Exception:
|
|
58
|
+
return cur_version, True
|
|
64
59
|
|
|
65
60
|
BEGIN_DELIM = "!{$"
|
|
66
61
|
END_DELIM_ENCODED = "!}$".encode('utf-8')
|
|
@@ -81,6 +76,12 @@ args = parser.parse_args()
|
|
|
81
76
|
# check basepath is readable
|
|
82
77
|
assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
|
|
83
78
|
|
|
79
|
+
# Keep startup snappy for daemon/diagnostics; check updates only in interactive mode.
|
|
80
|
+
if args.master_only or args.diag:
|
|
81
|
+
newest_version, updated = version_number, True
|
|
82
|
+
else:
|
|
83
|
+
newest_version, updated = check_latest_version(version_number)
|
|
84
|
+
|
|
84
85
|
# export args
|
|
85
86
|
orig_instance = Singleton.getInstance(args)
|
|
86
87
|
|
|
@@ -121,7 +122,7 @@ def send_msg(instance: Singleton, msg, n_extra_msgs=0, is_extra=False):
|
|
|
121
122
|
msg = (str(msg_len) + begin_delim).encode('utf-8') + msg + END_DELIM_ENCODED
|
|
122
123
|
instance.timeme(f"- msg encoded with len {msg_len}")
|
|
123
124
|
|
|
124
|
-
instance.sock
|
|
125
|
+
send_datagram(instance.sock, instance.port, msg)
|
|
125
126
|
|
|
126
127
|
|
|
127
128
|
def update_data_master(instance: Singleton):
|
|
@@ -139,24 +140,11 @@ def update_data_master(instance: Singleton):
|
|
|
139
140
|
running_jobs = [j.to_nested_dict() for j in jobs if j.state == 'CG' or j.state == 'R']
|
|
140
141
|
queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
|
|
141
142
|
queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
|
|
142
|
-
maxlen = min(JOB_LIMIT_PER_MSG, len(running_jobs) + len(queue_jobs))
|
|
143
|
-
|
|
144
|
-
queue_jobs, extra_queue_jobs = queue_jobs[:maxlen - len(running_jobs)], queue_jobs[maxlen - len(running_jobs):]
|
|
145
143
|
cur_timestamp = time.time()
|
|
146
144
|
msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs, 'ts': str(cur_timestamp)})
|
|
147
|
-
|
|
148
|
-
N_EXTRA_MSGS = (len(extra_queue_jobs) // JOB_LIMIT_PER_MSG) + 1 if len(extra_queue_jobs) > 0 else 0
|
|
149
|
-
send_msg(instance, msg, n_extra_msgs=N_EXTRA_MSGS)
|
|
145
|
+
send_msg(instance, msg)
|
|
150
146
|
instance.timeme("- main broadcast")
|
|
151
147
|
|
|
152
|
-
if extra_queue_jobs:
|
|
153
|
-
# get batches of JOB_LIMIT_PER_MSG jobs
|
|
154
|
-
for job_idx in range(N_EXTRA_MSGS):
|
|
155
|
-
i = job_idx * JOB_LIMIT_PER_MSG
|
|
156
|
-
msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': extra_queue_jobs[i:i + JOB_LIMIT_PER_MSG], 'ts': str(cur_timestamp)})
|
|
157
|
-
send_msg(instance, msg, is_extra=True, n_extra_msgs=N_EXTRA_MSGS - job_idx - 1)
|
|
158
|
-
instance.timeme(f"- extra broadcast #{job_idx+1}")
|
|
159
|
-
|
|
160
148
|
return inf, jobs
|
|
161
149
|
|
|
162
150
|
|
|
@@ -178,7 +166,7 @@ async def get_data_slave(instance):
|
|
|
178
166
|
try:
|
|
179
167
|
# listen for data from master
|
|
180
168
|
# instance.sock.settimeout(6.5)
|
|
181
|
-
data
|
|
169
|
+
data = receive_datagram(instance.sock, MAX_BUF)
|
|
182
170
|
decoded_data = data.decode('utf-8', errors='replace')
|
|
183
171
|
if BEGIN_DELIM not in decoded_data:
|
|
184
172
|
if EXTRA_MSG_BEGIN_DELIM in decoded_data:
|
|
@@ -198,7 +186,7 @@ async def get_data_slave(instance):
|
|
|
198
186
|
total_msg_n = n_msgs_remaining
|
|
199
187
|
msg_store = {i: [] for i in range(total_msg_n)}
|
|
200
188
|
while n_msgs_remaining > 0:
|
|
201
|
-
data
|
|
189
|
+
data = receive_datagram(instance.sock, MAX_BUF)
|
|
202
190
|
decoded_extra = data.decode('utf-8', errors='replace')
|
|
203
191
|
if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
|
|
204
192
|
instance.log("Received non-extra packet while waiting for extra packets, skipping")
|
|
@@ -239,8 +227,6 @@ async def get_data_slave(instance):
|
|
|
239
227
|
if n_timeout > TIMEOUT_MAX:
|
|
240
228
|
try_open_socket_as_slave(instance, force=True)
|
|
241
229
|
|
|
242
|
-
time.sleep(random.randint(100, 10000) / 1000)
|
|
243
|
-
|
|
244
230
|
except Exception as e:
|
|
245
231
|
instance.err(f"Exception while receiving data from master: {e}")
|
|
246
232
|
instance.err(traceback.format_exc())
|
|
@@ -272,7 +258,6 @@ async def get_all():
|
|
|
272
258
|
if orig_instance.is_master:
|
|
273
259
|
inf, jobs = update_data_master(orig_instance)
|
|
274
260
|
|
|
275
|
-
time.sleep(5)
|
|
276
261
|
else:
|
|
277
262
|
# loop = asyncio.get_event_loop()
|
|
278
263
|
orig_instance.timeme(f"- listening for data")
|
|
@@ -345,6 +330,7 @@ def _main():
|
|
|
345
330
|
orig_instance.log(f"Starting master daemon")
|
|
346
331
|
|
|
347
332
|
while True:
|
|
333
|
+
cycle_start = time.monotonic()
|
|
348
334
|
orig_instance.timeme(f"Updating...")
|
|
349
335
|
try:
|
|
350
336
|
update_data_master(orig_instance)
|
|
@@ -352,7 +338,7 @@ def _main():
|
|
|
352
338
|
orig_instance.err(f"Exception: {e}")
|
|
353
339
|
orig_instance.err(traceback.format_exc())
|
|
354
340
|
|
|
355
|
-
time.sleep(5)
|
|
341
|
+
time.sleep(max(0, 5 - (time.monotonic() - cycle_start)))
|
|
356
342
|
|
|
357
343
|
else:
|
|
358
344
|
# configure singleton
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nodeocc
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.27
|
|
4
4
|
Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
|
|
5
5
|
Author: Matteo Boschini
|
|
6
6
|
Maintainer: Lorenzo Bonicelli, Aniello Panariello
|
|
@@ -31,4 +31,8 @@ Contents:
|
|
|
31
31
|
|
|
32
32
|
Run it with: `python controller/controller.py`
|
|
33
33
|
|
|
34
|
+
Press `D` in the interactive view to open or close the debug screen. It shows
|
|
35
|
+
the local role, master PID and port, snapshot status, and local package
|
|
36
|
+
versions. Use the up and down keys to scroll.
|
|
37
|
+
|
|
34
38
|
Requirements: pandas, pycurses.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import re
|
|
3
|
+
from collections import defaultdict
|
|
3
4
|
from io import StringIO
|
|
4
5
|
|
|
5
6
|
import pandas as pd
|
|
@@ -57,9 +58,16 @@ def _parse_gpu_count(gres):
|
|
|
57
58
|
token = token.strip()
|
|
58
59
|
if not token.lower().startswith('gpu'):
|
|
59
60
|
continue
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
token_core = token.split('(')[0]
|
|
62
|
+
parts = token_core.split(':')
|
|
63
|
+
count = None
|
|
64
|
+
|
|
65
|
+
if len(parts) >= 2 and re.fullmatch(r'\d+', parts[-1]):
|
|
66
|
+
count = int(parts[-1])
|
|
67
|
+
elif len(parts) >= 3 and re.fullmatch(r'\d+', parts[-2]):
|
|
68
|
+
count = int(parts[-2])
|
|
69
|
+
|
|
70
|
+
n_gpu += count if count is not None else 1
|
|
63
71
|
return n_gpu
|
|
64
72
|
|
|
65
73
|
|
|
@@ -75,10 +83,13 @@ def _parse_gpu_models(gres):
|
|
|
75
83
|
token = token.strip()
|
|
76
84
|
if not token.lower().startswith('gpu'):
|
|
77
85
|
continue
|
|
78
|
-
|
|
79
|
-
|
|
86
|
+
token_core = token.split('(')[0]
|
|
87
|
+
parts = token_core.split(':')
|
|
88
|
+
if len(parts) >= 3 and not re.fullmatch(r'\d+', parts[1]):
|
|
89
|
+
model = parts[1]
|
|
90
|
+
elif len(parts) == 2 and not re.fullmatch(r'\d+', parts[1]):
|
|
80
91
|
model = parts[1]
|
|
81
|
-
elif len(parts) == 2:
|
|
92
|
+
elif len(parts) == 2 and re.fullmatch(r'\d+', parts[1]):
|
|
82
93
|
model = 'gpu'
|
|
83
94
|
else:
|
|
84
95
|
model = 'n/a'
|
|
@@ -367,8 +378,11 @@ def read_jobs():
|
|
|
367
378
|
line['NICE']
|
|
368
379
|
), axis=1).tolist()
|
|
369
380
|
|
|
381
|
+
joblets_by_job = defaultdict(list)
|
|
382
|
+
for joblet in joblets:
|
|
383
|
+
joblets_by_job[joblet.jobid].append(joblet)
|
|
370
384
|
for j in jobs:
|
|
371
|
-
j.joblets = [
|
|
385
|
+
j.joblets = joblets_by_job[j.jobid]
|
|
372
386
|
return jobs, joblets
|
|
373
387
|
|
|
374
388
|
|
|
@@ -13,5 +13,11 @@ def update_views(stdscr, instance, filter):
|
|
|
13
13
|
instance, 'job_id_type') else 'agg')
|
|
14
14
|
instance.timeme(f"- print job list")
|
|
15
15
|
|
|
16
|
-
instance.nocc = slurm_viz.view_viz(
|
|
16
|
+
instance.nocc = slurm_viz.view_viz(
|
|
17
|
+
instance.inf,
|
|
18
|
+
instance.jobs,
|
|
19
|
+
stylefn=styles.crsstyler,
|
|
20
|
+
mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu',
|
|
21
|
+
panel_width=instance.right_width if hasattr(instance, 'right_width') else None,
|
|
22
|
+
)
|
|
17
23
|
instance.timeme(f"- view nodeocc")
|
|
@@ -14,6 +14,9 @@ from logging.handlers import RotatingFileHandler
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
from view import update_views
|
|
16
16
|
import stat
|
|
17
|
+
import sys
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from importlib import metadata
|
|
17
20
|
|
|
18
21
|
a_filter_values = [None, 'me', 'prod', 'stud', 'cvcs']
|
|
19
22
|
|
|
@@ -33,8 +36,14 @@ BUTTON_ACTIONS = {
|
|
|
33
36
|
'show_prio': [ord('p')],
|
|
34
37
|
'refresh': [ord('y')],
|
|
35
38
|
'info': [ord('i')],
|
|
39
|
+
'debug_view': [ord('D')],
|
|
36
40
|
}
|
|
37
41
|
|
|
42
|
+
RIGHT_WIDTH_GPU = 33
|
|
43
|
+
RIGHT_WIDTH_INFO = 39
|
|
44
|
+
# Keep a single stable width across modes; use the widest one so info fields are not cut.
|
|
45
|
+
RIGHT_WIDTH_UNIFIED = max(RIGHT_WIDTH_GPU, RIGHT_WIDTH_INFO)
|
|
46
|
+
|
|
38
47
|
|
|
39
48
|
def is_file_writable_byall(file_path):
|
|
40
49
|
return os.stat(file_path).st_mode & stat.S_IWOTH
|
|
@@ -48,7 +57,10 @@ def is_nodeocc_process(pid):
|
|
|
48
57
|
try:
|
|
49
58
|
cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
|
|
50
59
|
return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
|
|
51
|
-
except
|
|
60
|
+
except psutil.AccessDenied:
|
|
61
|
+
# Other users' command lines may be hidden on shared login nodes.
|
|
62
|
+
return psutil.pid_exists(int(pid))
|
|
63
|
+
except (psutil.Error, ValueError):
|
|
52
64
|
return False
|
|
53
65
|
|
|
54
66
|
|
|
@@ -67,7 +79,10 @@ def process_has_udp_port(pid, port):
|
|
|
67
79
|
for conn in proc.connections(kind='udp'):
|
|
68
80
|
if conn.laddr and conn.laddr.port == int(port):
|
|
69
81
|
return True
|
|
70
|
-
except
|
|
82
|
+
except psutil.AccessDenied:
|
|
83
|
+
# Socket ownership is commonly visible only to the process owner.
|
|
84
|
+
return psutil.pid_exists(int(pid))
|
|
85
|
+
except (psutil.Error, ValueError):
|
|
71
86
|
return False
|
|
72
87
|
return False
|
|
73
88
|
|
|
@@ -169,6 +184,11 @@ class Singleton:
|
|
|
169
184
|
self.rens = ""
|
|
170
185
|
|
|
171
186
|
self.fetch_fn = None
|
|
187
|
+
self.debug_view = False
|
|
188
|
+
self.debug_offset = 0
|
|
189
|
+
self.last_success = None
|
|
190
|
+
self.last_fetch_duration = None
|
|
191
|
+
self.last_error = None
|
|
172
192
|
self.view_mode = 'gpu'
|
|
173
193
|
self.job_id_type = 'agg'
|
|
174
194
|
|
|
@@ -319,6 +339,7 @@ class Singleton:
|
|
|
319
339
|
self._ctime = _ntime
|
|
320
340
|
|
|
321
341
|
def err(self, msg):
|
|
342
|
+
self.last_error = str(msg)
|
|
322
343
|
if self.args.debug or self.args.master_only:
|
|
323
344
|
logging.error(msg)
|
|
324
345
|
|
|
@@ -332,11 +353,14 @@ class Singleton:
|
|
|
332
353
|
if self.fetch_fn is not None:
|
|
333
354
|
inf, jobs, avg_wait_time = await self.fetch_fn() # a_filter_values[self.a_filter])
|
|
334
355
|
master_dead = inf is None
|
|
356
|
+
if not master_dead:
|
|
357
|
+
self.last_success = time.time()
|
|
335
358
|
self.inf = inf if inf is not None else self.inf
|
|
336
359
|
self.jobs = jobs if jobs is not None else self.jobs
|
|
337
360
|
self.avg_wait_time = avg_wait_time if avg_wait_time is not None else self.avg_wait_time
|
|
338
361
|
|
|
339
362
|
_delta_t = time.time() - _ctime
|
|
363
|
+
self.last_fetch_duration = _delta_t
|
|
340
364
|
self.log(f"Fetch took {_delta_t:.2f} seconds")
|
|
341
365
|
return master_dead
|
|
342
366
|
|
|
@@ -454,6 +478,18 @@ def handle_keys(stdscr, instance):
|
|
|
454
478
|
|
|
455
479
|
instance.mouse_state = {}
|
|
456
480
|
|
|
481
|
+
if k in BUTTON_ACTIONS['debug_view']:
|
|
482
|
+
instance.debug_view = not instance.debug_view
|
|
483
|
+
instance.debug_offset = 0
|
|
484
|
+
stdscr.clear()
|
|
485
|
+
return
|
|
486
|
+
if instance.debug_view:
|
|
487
|
+
if k in BUTTON_ACTIONS['down_1']:
|
|
488
|
+
instance.debug_offset += 1
|
|
489
|
+
elif k in BUTTON_ACTIONS['up_1']:
|
|
490
|
+
instance.debug_offset = max(0, instance.debug_offset - 1)
|
|
491
|
+
return
|
|
492
|
+
|
|
457
493
|
# process input
|
|
458
494
|
# RIGHT
|
|
459
495
|
if k in BUTTON_ACTIONS['right_1']:
|
|
@@ -476,7 +512,7 @@ def handle_keys(stdscr, instance):
|
|
|
476
512
|
instance.sort_by_prio = not instance.sort_by_prio
|
|
477
513
|
elif k in BUTTON_ACTIONS['res_view_mode']:
|
|
478
514
|
instance.view_mode = {"gpu": "ram", "ram": "cpu", "cpu": "gpu", "info": "gpu"}[instance.view_mode]
|
|
479
|
-
instance.right_width =
|
|
515
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
480
516
|
stdscr.clear()
|
|
481
517
|
elif k in BUTTON_ACTIONS['job_id_type']:
|
|
482
518
|
instance.job_id_type = "true" if instance.job_id_type == "agg" else "agg"
|
|
@@ -489,7 +525,7 @@ def handle_keys(stdscr, instance):
|
|
|
489
525
|
instance.show_prio = not instance.show_prio
|
|
490
526
|
elif k in BUTTON_ACTIONS['info']:
|
|
491
527
|
instance.view_mode = 'info' if instance.view_mode != 'info' else 'gpu'
|
|
492
|
-
instance.right_width =
|
|
528
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
493
529
|
stdscr.clear()
|
|
494
530
|
elif k in BUTTON_ACTIONS['up_tab']:
|
|
495
531
|
# get screen size
|
|
@@ -502,7 +538,70 @@ def handle_keys(stdscr, instance):
|
|
|
502
538
|
instance.voff -= ((height - 4) // 2) + 1
|
|
503
539
|
|
|
504
540
|
|
|
541
|
+
def debug_lines(instance):
|
|
542
|
+
"""Collect local diagnostics without querying Slurm or the network."""
|
|
543
|
+
if not hasattr(instance, '_package_versions'):
|
|
544
|
+
instance._package_versions = {}
|
|
545
|
+
for name in ('nodeocc', 'pandas', 'numpy', 'psutil'):
|
|
546
|
+
try:
|
|
547
|
+
instance._package_versions[name] = metadata.version(name)
|
|
548
|
+
except metadata.PackageNotFoundError:
|
|
549
|
+
instance._package_versions[name] = 'not installed'
|
|
550
|
+
port = getattr(instance, 'port', None)
|
|
551
|
+
master_pid = str(os.getpid()) if instance.is_master else 'unknown'
|
|
552
|
+
try:
|
|
553
|
+
for path in instance.get_port_files():
|
|
554
|
+
if port_from_portfile(path) == port:
|
|
555
|
+
master_pid = Path(path).read_text().strip() or 'unknown'
|
|
556
|
+
break
|
|
557
|
+
except OSError:
|
|
558
|
+
pass
|
|
559
|
+
try:
|
|
560
|
+
master_owner = psutil.Process(int(master_pid)).username()
|
|
561
|
+
except (ValueError, psutil.Error, OSError, KeyError):
|
|
562
|
+
master_owner = 'unknown'
|
|
563
|
+
last_success = (datetime.fromtimestamp(instance.last_success).strftime('%Y-%m-%d %H:%M:%S')
|
|
564
|
+
if instance.last_success else 'never')
|
|
565
|
+
duration = (f'{instance.last_fetch_duration:.2f} s'
|
|
566
|
+
if instance.last_fetch_duration is not None else 'not available')
|
|
567
|
+
lines = [
|
|
568
|
+
'Connection',
|
|
569
|
+
f' Host: {socket.getfqdn()}',
|
|
570
|
+
f' Role: {"master" if instance.is_master else "client"}',
|
|
571
|
+
f' Master PID: {master_pid}',
|
|
572
|
+
f' Master owner: {master_owner}',
|
|
573
|
+
f' Master UDP port: {port if port is not None else "unknown"}',
|
|
574
|
+
f' Base path: {instance.basepath}',
|
|
575
|
+
'', 'Snapshot',
|
|
576
|
+
f' Last successful update: {last_success}',
|
|
577
|
+
f' Last {"update" if instance.is_master else "receive wait"}: {duration}',
|
|
578
|
+
f' Jobs: {len(instance.jobs) if instance.jobs is not None else "unknown"}',
|
|
579
|
+
f' Nodes: {len(instance.inf.nodes) if instance.inf is not None else "unknown"}',
|
|
580
|
+
f' Last error: {instance.last_error or "none recorded"}',
|
|
581
|
+
'', 'Environment',
|
|
582
|
+
f' Python: {sys.version.split()[0]}',
|
|
583
|
+
f' Executable: {sys.executable}',
|
|
584
|
+
f' Process PID: {os.getpid()}',
|
|
585
|
+
' Package versions (local environment):',
|
|
586
|
+
]
|
|
587
|
+
lines.extend(f' {name}: {version}' for name, version in instance._package_versions.items())
|
|
588
|
+
return lines
|
|
589
|
+
|
|
590
|
+
|
|
505
591
|
def update_screen(stdscr, instance):
|
|
592
|
+
if instance.debug_view:
|
|
593
|
+
lines, columns = stdscr.getmaxyx()
|
|
594
|
+
content = debug_lines(instance)
|
|
595
|
+
instance.debug_offset = min(instance.debug_offset, max(0, len(content) - max(1, lines - 2)))
|
|
596
|
+
stdscr.erase()
|
|
597
|
+
try:
|
|
598
|
+
stdscr.addnstr(0, 0, 'DEBUG [D: close] [up/down: scroll] [Q: quit]', max(0, columns - 1))
|
|
599
|
+
for row, line in enumerate(content[instance.debug_offset:instance.debug_offset + max(0, lines - 2)], 1):
|
|
600
|
+
stdscr.addnstr(row, 0, line, max(0, columns - 1))
|
|
601
|
+
except curses.error:
|
|
602
|
+
pass
|
|
603
|
+
stdscr.refresh()
|
|
604
|
+
return
|
|
506
605
|
update_views(stdscr, instance, a_filter_values[instance.a_filter])
|
|
507
606
|
xoffset = 0
|
|
508
607
|
instance.xoffset = xoffset
|
|
@@ -513,7 +612,7 @@ def update_screen(stdscr, instance):
|
|
|
513
612
|
if instance.k in BUTTON_ACTIONS['refresh']:
|
|
514
613
|
stdscr.clear()
|
|
515
614
|
|
|
516
|
-
totsize = 106
|
|
615
|
+
totsize = 106 + max(0, RIGHT_WIDTH_UNIFIED - RIGHT_WIDTH_GPU)
|
|
517
616
|
if instance.show_account:
|
|
518
617
|
totsize += 10
|
|
519
618
|
if instance.show_prio:
|
|
@@ -579,7 +678,9 @@ def update_screen(stdscr, instance):
|
|
|
579
678
|
stdscr.addstr(0, left_width + 5, 'INFO', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'info' else 0))
|
|
580
679
|
stdscr.addstr(0, left_width + 9, ']', curses.color_pair(2))
|
|
581
680
|
instance.add_button(0, left_width + 2, '[I:INFO]', BUTTON_ACTIONS['info'][0])
|
|
582
|
-
stdscr.addstr(0, left_width +
|
|
681
|
+
stdscr.addstr(0, left_width + 11, '[D:DEBUG]', curses.color_pair(2))
|
|
682
|
+
instance.add_button(0, left_width + 11, '[D:DEBUG]', BUTTON_ACTIONS['debug_view'][0])
|
|
683
|
+
stdscr.addstr(0, left_width + 20, '─' * max(0, columns - 15 - left_width - 20), curses.color_pair(2))
|
|
583
684
|
stdscr.addstr(0, columns - 15, '[G:', curses.color_pair(2))
|
|
584
685
|
stdscr.addstr(0, columns - 12, 'GPU', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'gpu' else 0))
|
|
585
686
|
stdscr.addstr(0, columns - 12 + 3, 'RAM', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'ram' else 0))
|
|
@@ -606,8 +707,14 @@ def update_screen(stdscr, instance):
|
|
|
606
707
|
# get slurm user partition
|
|
607
708
|
# stdscr.addstr(lines - 1, xoffset + 62 + 2, f'(Avg time {instance.avg_wait_time})', curses.color_pair(2))
|
|
608
709
|
|
|
609
|
-
|
|
610
|
-
|
|
710
|
+
if instance.updated:
|
|
711
|
+
signature = instance.signature
|
|
712
|
+
sig_style = curses.color_pair(2)
|
|
713
|
+
else:
|
|
714
|
+
signature = f'{instance.signature} (new {instance.newest_version})'
|
|
715
|
+
sig_style = curses.color_pair(4)
|
|
716
|
+
|
|
717
|
+
stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, sig_style)
|
|
611
718
|
|
|
612
719
|
stdscr.refresh()
|
|
613
720
|
curses.doupdate()
|
|
@@ -645,6 +752,8 @@ async def update_screen_info(stdscr, instance: Singleton):
|
|
|
645
752
|
try_open_socket_as_slave(instance)
|
|
646
753
|
|
|
647
754
|
update_screen(stdscr, instance)
|
|
755
|
+
if instance.is_master:
|
|
756
|
+
await asyncio.sleep(max(0, 5 - (instance.last_fetch_duration or 0)))
|
|
648
757
|
|
|
649
758
|
|
|
650
759
|
async def wait_first(futures, instance):
|
|
@@ -712,7 +821,7 @@ async def main(stdscr):
|
|
|
712
821
|
# status
|
|
713
822
|
|
|
714
823
|
instance.a_filter = 0
|
|
715
|
-
instance.right_width =
|
|
824
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
716
825
|
|
|
717
826
|
stdscr.clear()
|
|
718
827
|
|
|
@@ -44,7 +44,7 @@ def _joblet_format(instance, job, width=74, jobid_type='agg'):
|
|
|
44
44
|
mem = round(joblet.mem / 1024)
|
|
45
45
|
joblet_repr += _format_to(f'{mem}G', 4, 'right')
|
|
46
46
|
joblet_repr += ' '
|
|
47
|
-
if instance.view_mode in ('gpu', 'ram'):
|
|
47
|
+
if instance.view_mode in ('gpu', 'ram', 'info'):
|
|
48
48
|
joblet_repr += _format_to((f'{joblet.n_gpus}gp' if joblet.n_gpus > 0 else " - "), 3, 'left')
|
|
49
49
|
elif instance.view_mode == 'cpu':
|
|
50
50
|
joblet_repr += _format_to((f'{joblet.cpus}cp' if joblet.cpus > 0 else " - "), 3, 'left')
|
|
@@ -101,6 +101,20 @@ cpu_paused = '▚'
|
|
|
101
101
|
cpu_down = '⨯'
|
|
102
102
|
|
|
103
103
|
GLOBAL_NODES = None
|
|
104
|
+
BASE_PANEL_WIDTH = 33
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _center_panel_content(text, panel_width, base_width=BASE_PANEL_WIDTH):
|
|
108
|
+
if panel_width is None or panel_width <= base_width:
|
|
109
|
+
return text
|
|
110
|
+
|
|
111
|
+
pad = max(0, (panel_width - base_width) // 2)
|
|
112
|
+
if pad == 0:
|
|
113
|
+
return text
|
|
114
|
+
|
|
115
|
+
lines = text.split('\n')
|
|
116
|
+
lines = [(' ' * pad + line) if line else line for line in lines]
|
|
117
|
+
return '\n'.join(lines)
|
|
104
118
|
|
|
105
119
|
|
|
106
120
|
def get_ram_block(megs):
|
|
@@ -138,15 +152,15 @@ def get_node_status(node, jobs):
|
|
|
138
152
|
return None # white?
|
|
139
153
|
|
|
140
154
|
|
|
141
|
-
def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu'):
|
|
155
|
+
def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu', panel_width=None):
|
|
142
156
|
if mode == 'gpu':
|
|
143
|
-
return view_viz_gpu(infrastructure, jobs, work, stylefn, current_user)
|
|
157
|
+
return _center_panel_content(view_viz_gpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
144
158
|
elif mode == 'ram':
|
|
145
|
-
return view_viz_ram(infrastructure, jobs, work, stylefn, current_user)
|
|
159
|
+
return _center_panel_content(view_viz_ram(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
146
160
|
elif mode == 'cpu':
|
|
147
|
-
return view_viz_cpu(infrastructure, jobs, work, stylefn, current_user)
|
|
161
|
+
return _center_panel_content(view_viz_cpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
148
162
|
elif mode == 'info':
|
|
149
|
-
return view_viz_info(infrastructure, jobs, work, stylefn, current_user)
|
|
163
|
+
return view_viz_info(infrastructure, jobs, work, stylefn, current_user, panel_width)
|
|
150
164
|
|
|
151
165
|
|
|
152
166
|
def view_viz_ram(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
|
|
@@ -519,7 +533,7 @@ def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user
|
|
|
519
533
|
return RetScope.return_string
|
|
520
534
|
|
|
521
535
|
|
|
522
|
-
def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
|
|
536
|
+
def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, panel_width=None):
|
|
523
537
|
# this is for hot reload
|
|
524
538
|
if not work:
|
|
525
539
|
return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
|
|
@@ -555,6 +569,7 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
|
|
|
555
569
|
# max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
|
|
556
570
|
feat_str = stylefn("GREEN", "FEATURES")
|
|
557
571
|
RetScope.return_string += f'{"":{gpu_name_chars}} GPU {feat_str}\n'
|
|
572
|
+
|
|
558
573
|
for i, n in GLOBAL_NODES.iterrows():
|
|
559
574
|
gpu_model = n.m_gpu # if n.m_gpu is not None else 'N/A'
|
|
560
575
|
# gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Fragment large broadcast messages to fit in UDP datagrams."""
|
|
2
|
+
import uuid
|
|
3
|
+
import zlib
|
|
4
|
+
|
|
5
|
+
PREFIX = b'nodeocc-fragment:'
|
|
6
|
+
CHUNK_SIZE = 60000
|
|
7
|
+
COMPRESSED = b'nodeocc-zlib:'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def send(sock, port, data):
|
|
11
|
+
compressed = COMPRESSED + zlib.compress(data, level=1)
|
|
12
|
+
if len(compressed) < len(data):
|
|
13
|
+
data = compressed
|
|
14
|
+
if len(data) <= CHUNK_SIZE:
|
|
15
|
+
sock.sendto(data, ('<broadcast>', port))
|
|
16
|
+
return
|
|
17
|
+
message_id = uuid.uuid4().hex.encode('ascii')
|
|
18
|
+
count = (len(data) + CHUNK_SIZE - 1) // CHUNK_SIZE
|
|
19
|
+
for index in range(count):
|
|
20
|
+
header = PREFIX + message_id + b':' + str(index).encode() + b':' + str(count).encode() + b':'
|
|
21
|
+
sock.sendto(header + data[index * CHUNK_SIZE:(index + 1) * CHUNK_SIZE], ('<broadcast>', port))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def receive(sock, max_buf):
|
|
25
|
+
fragments = {}
|
|
26
|
+
while True:
|
|
27
|
+
data, _ = sock.recvfrom(max_buf)
|
|
28
|
+
if not data.startswith(PREFIX):
|
|
29
|
+
return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data
|
|
30
|
+
try:
|
|
31
|
+
message_id, index, count, chunk = data[len(PREFIX):].split(b':', 3)
|
|
32
|
+
index, count = int(index), int(count)
|
|
33
|
+
if count < 2 or count > 10000 or not 0 <= index < count:
|
|
34
|
+
continue
|
|
35
|
+
except ValueError:
|
|
36
|
+
continue
|
|
37
|
+
# Retain only one message, so lost fragments cannot accumulate forever.
|
|
38
|
+
if message_id not in fragments:
|
|
39
|
+
fragments = {message_id: (count, {})}
|
|
40
|
+
expected, parts = fragments[message_id]
|
|
41
|
+
if count != expected:
|
|
42
|
+
continue
|
|
43
|
+
parts[index] = chunk
|
|
44
|
+
if len(parts) == count:
|
|
45
|
+
data = b''.join(parts[i] for i in range(count))
|
|
46
|
+
return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|