nodeocc 1.0.24__tar.gz → 1.0.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nodeocc-1.0.24 → nodeocc-1.0.26}/PKG-INFO +2 -2
- {nodeocc-1.0.24 → nodeocc-1.0.26}/pyproject.toml +1 -1
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/controller/controller.py +132 -68
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/PKG-INFO +2 -2
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/readers/slurmreader.py +51 -11
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/__init__.py +9 -1
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/curses_multiwindow.py +91 -30
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/slurm_list.py +1 -1
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/slurm_viz.py +21 -6
- {nodeocc-1.0.24 → nodeocc-1.0.26}/README.md +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/setup.cfg +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/setup.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/__init__.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/model/infrastructure.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/model/job.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/SOURCES.txt +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/dependency_links.txt +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/entry_points.txt +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/requires.txt +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/top_level.txt +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/readers/compute_wait_time.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/styles.py +0 -0
- {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/utils.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "nodeocc"
|
|
7
|
-
version = "1.0.
|
|
7
|
+
version = "1.0.26"
|
|
8
8
|
authors = [{ name = "Matteo Boschini" }]
|
|
9
9
|
maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
|
|
10
10
|
description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"
|
|
@@ -8,7 +8,7 @@ import numpy as np
|
|
|
8
8
|
|
|
9
9
|
conf_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
10
10
|
print(conf_path)
|
|
11
|
-
sys.path.
|
|
11
|
+
sys.path.insert(0, conf_path)
|
|
12
12
|
from curses import wrapper
|
|
13
13
|
from view.curses_multiwindow import main, Singleton, try_open_socket_as_slave
|
|
14
14
|
|
|
@@ -23,6 +23,9 @@ import time
|
|
|
23
23
|
import json
|
|
24
24
|
import argparse
|
|
25
25
|
import traceback
|
|
26
|
+
import socket
|
|
27
|
+
import re
|
|
28
|
+
from urllib.request import urlopen
|
|
26
29
|
from model.infrastructure import Infrastructure
|
|
27
30
|
from model.job import Job
|
|
28
31
|
import asyncio
|
|
@@ -30,36 +33,28 @@ import importlib.metadata
|
|
|
30
33
|
|
|
31
34
|
program_name = 'nodeocc'
|
|
32
35
|
version_number = importlib.metadata.version(program_name)
|
|
33
|
-
updated = False
|
|
34
36
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
updated = True
|
|
57
|
-
time.sleep(3)
|
|
58
|
-
print(END, end='')
|
|
59
|
-
except Exception as e:
|
|
60
|
-
print(f"{BBLUE}Could not get newest version from pip, check if {program_name} is up to date{END}")
|
|
61
|
-
newest_version = '?.?.?'
|
|
62
|
-
time.sleep(5)
|
|
37
|
+
|
|
38
|
+
def _version_tuple(v):
|
|
39
|
+
return tuple(int(x) for x in re.findall(r'\d+', str(v)))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def check_latest_version(cur_version, timeout_sec=1.2):
|
|
43
|
+
"""
|
|
44
|
+
Fast, non-blocking-ish version check:
|
|
45
|
+
- short timeout
|
|
46
|
+
- no startup sleeps/pauses
|
|
47
|
+
- graceful fallback to current version on errors
|
|
48
|
+
"""
|
|
49
|
+
try:
|
|
50
|
+
url = f"https://pypi.python.org/pypi/{program_name}/json"
|
|
51
|
+
with urlopen(url, timeout=timeout_sec) as response:
|
|
52
|
+
payload = json.loads(response.read().decode('utf-8'))
|
|
53
|
+
newest = payload['info']['version']
|
|
54
|
+
is_updated = _version_tuple(cur_version) >= _version_tuple(newest)
|
|
55
|
+
return newest, is_updated
|
|
56
|
+
except Exception:
|
|
57
|
+
return cur_version, True
|
|
63
58
|
|
|
64
59
|
BEGIN_DELIM = "!{$"
|
|
65
60
|
END_DELIM_ENCODED = "!}$".encode('utf-8')
|
|
@@ -74,11 +69,18 @@ parser.add_argument('--debug', action='store_true', help='Enable logging')
|
|
|
74
69
|
parser.add_argument('--override', action='store_true', help='Enable logging')
|
|
75
70
|
parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
|
|
76
71
|
parser.add_argument('--basepath', type=str, default='/tmp', help='Base path for nodeocc. Must be readable by all users')
|
|
72
|
+
parser.add_argument('--diag', action='store_true', help='Run one-shot diagnostics and exit')
|
|
77
73
|
args = parser.parse_args()
|
|
78
74
|
|
|
79
75
|
# check basepath is readable
|
|
80
76
|
assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
|
|
81
77
|
|
|
78
|
+
# Keep startup snappy for daemon/diagnostics; check updates only in interactive mode.
|
|
79
|
+
if args.master_only or args.diag:
|
|
80
|
+
newest_version, updated = version_number, True
|
|
81
|
+
else:
|
|
82
|
+
newest_version, updated = check_latest_version(version_number)
|
|
83
|
+
|
|
82
84
|
# export args
|
|
83
85
|
orig_instance = Singleton.getInstance(args)
|
|
84
86
|
|
|
@@ -177,49 +179,61 @@ async def get_data_slave(instance):
|
|
|
177
179
|
# listen for data from master
|
|
178
180
|
# instance.sock.settimeout(6.5)
|
|
179
181
|
data, _ = instance.sock.recvfrom(MAX_BUF)
|
|
180
|
-
decoded_data = data.decode('utf-8')
|
|
181
|
-
if BEGIN_DELIM in decoded_data:
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
182
|
+
decoded_data = data.decode('utf-8', errors='replace')
|
|
183
|
+
if BEGIN_DELIM not in decoded_data:
|
|
184
|
+
if EXTRA_MSG_BEGIN_DELIM in decoded_data:
|
|
185
|
+
instance.log("Received extra packet before main packet, discarding")
|
|
186
|
+
else:
|
|
187
|
+
instance.log("Received packet without known delimiters, discarding")
|
|
188
|
+
return None, None, None
|
|
189
|
+
|
|
190
|
+
n_msgs_remaining = int(decoded_data.split(BEGIN_DELIM)[0])
|
|
191
|
+
# msg_len += len(str(msg_len)) message len is deprecated and used for n_msgs_remaining
|
|
192
|
+
instance.timeme(f"received first message of {n_msgs_remaining} (total len {len(data)} bytes)")
|
|
193
|
+
|
|
194
|
+
inf, jobs, avg_wait_time, decoded_msg = decode_msg_slave(data)
|
|
195
|
+
instance.timeme(f"- receive")
|
|
196
|
+
orig_timestamp = float(decoded_msg['ts'])
|
|
197
|
+
|
|
198
|
+
total_msg_n = n_msgs_remaining
|
|
199
|
+
msg_store = {i: [] for i in range(total_msg_n)}
|
|
200
|
+
while n_msgs_remaining > 0:
|
|
201
|
+
data, _ = instance.sock.recvfrom(MAX_BUF)
|
|
202
|
+
decoded_extra = data.decode('utf-8', errors='replace')
|
|
203
|
+
if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
|
|
204
|
+
instance.log("Received non-extra packet while waiting for extra packets, skipping")
|
|
205
|
+
continue
|
|
206
|
+
|
|
207
|
+
back_msg_idx = int(decoded_extra.split(EXTRA_MSG_BEGIN_DELIM)[0])
|
|
208
|
+
msg_idx = total_msg_n - back_msg_idx - 1
|
|
209
|
+
if msg_idx < 0 or msg_idx >= total_msg_n:
|
|
210
|
+
instance.log(f"Invalid extra packet index: {msg_idx}, skipping")
|
|
211
|
+
continue
|
|
212
|
+
|
|
213
|
+
_, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
|
|
214
|
+
|
|
215
|
+
# check if timestamp is the same
|
|
216
|
+
if float(new_ts['ts']) != orig_timestamp:
|
|
217
|
+
instance.timeme(f"Timestamp mismatch, skipping")
|
|
218
|
+
return inf, jobs, avg_wait_time # return what we have
|
|
219
|
+
|
|
220
|
+
msg_store[msg_idx] += new_jobs
|
|
221
|
+
|
|
222
|
+
n_msgs_remaining -= 1
|
|
223
|
+
instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
|
|
224
|
+
|
|
225
|
+
# sort and merge
|
|
226
|
+
for msg_idx in range(total_msg_n):
|
|
227
|
+
if len(msg_store[msg_idx]) == 0:
|
|
228
|
+
instance.log(f"Empty message {msg_idx}")
|
|
229
|
+
jobs += msg_store[msg_idx]
|
|
216
230
|
|
|
217
231
|
n_timeout = 0
|
|
218
232
|
|
|
219
|
-
except BlockingIOError
|
|
233
|
+
except BlockingIOError:
|
|
220
234
|
pass
|
|
221
235
|
|
|
222
|
-
except TimeoutError
|
|
236
|
+
except TimeoutError:
|
|
223
237
|
n_timeout += 1
|
|
224
238
|
instance.log(f"TIMEOUT {n_timeout}/{TIMEOUT_MAX}")
|
|
225
239
|
if n_timeout > TIMEOUT_MAX:
|
|
@@ -227,6 +241,10 @@ async def get_data_slave(instance):
|
|
|
227
241
|
|
|
228
242
|
time.sleep(random.randint(100, 10000) / 1000)
|
|
229
243
|
|
|
244
|
+
except Exception as e:
|
|
245
|
+
instance.err(f"Exception while receiving data from master: {e}")
|
|
246
|
+
instance.err(traceback.format_exc())
|
|
247
|
+
|
|
230
248
|
return inf, jobs, avg_wait_time
|
|
231
249
|
|
|
232
250
|
|
|
@@ -274,7 +292,53 @@ def display_main(stdscr):
|
|
|
274
292
|
return asyncio.run(main(stdscr))
|
|
275
293
|
|
|
276
294
|
|
|
295
|
+
def run_diagnostics(instance: Singleton):
|
|
296
|
+
hostname = socket.getfqdn().split('.')[0]
|
|
297
|
+
print(f"Diagnostics for nodeocc on host: {hostname}")
|
|
298
|
+
print(f"Base path: {instance.basepath}")
|
|
299
|
+
print(f"Role detected: {'master' if instance.is_master else 'slave'}")
|
|
300
|
+
print("")
|
|
301
|
+
|
|
302
|
+
try:
|
|
303
|
+
port_files = instance.get_port_files()
|
|
304
|
+
print(f"Visible port files: {len(port_files)}")
|
|
305
|
+
for pf in port_files:
|
|
306
|
+
try:
|
|
307
|
+
pid = Path(pf).read_text().strip()
|
|
308
|
+
except Exception:
|
|
309
|
+
pid = "?"
|
|
310
|
+
print(f" - {pf} (pid={pid})")
|
|
311
|
+
except Exception as e:
|
|
312
|
+
print(f"Could not list port files: {e}")
|
|
313
|
+
|
|
314
|
+
print("")
|
|
315
|
+
|
|
316
|
+
if instance.is_master:
|
|
317
|
+
print("Running one master update cycle...")
|
|
318
|
+
try:
|
|
319
|
+
inf, jobs = update_data_master(instance)
|
|
320
|
+
print(f"Master update OK. Nodes={len(inf.nodes)}, Jobs={len(jobs)}, Port={instance.port}")
|
|
321
|
+
except Exception as e:
|
|
322
|
+
print(f"Master update FAILED: {e}")
|
|
323
|
+
print(traceback.format_exc())
|
|
324
|
+
else:
|
|
325
|
+
print("Waiting for one packet from master...")
|
|
326
|
+
try:
|
|
327
|
+
inf, jobs, _ = asyncio.run(get_data_slave(instance))
|
|
328
|
+
if inf is None or jobs is None:
|
|
329
|
+
print("No valid payload received from master.")
|
|
330
|
+
else:
|
|
331
|
+
print(f"Slave receive OK. Jobs in snapshot={len(jobs)}")
|
|
332
|
+
except Exception as e:
|
|
333
|
+
print(f"Slave receive FAILED: {e}")
|
|
334
|
+
print(traceback.format_exc())
|
|
335
|
+
|
|
336
|
+
|
|
277
337
|
def _main():
|
|
338
|
+
if args.diag:
|
|
339
|
+
run_diagnostics(orig_instance)
|
|
340
|
+
return
|
|
341
|
+
|
|
278
342
|
if args.master_only:
|
|
279
343
|
assert orig_instance.is_master, "Instance is not `master`, run with `force_override` to override"
|
|
280
344
|
|
|
@@ -45,24 +45,64 @@ def _node_from_sinfo(x):
|
|
|
45
45
|
return n
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
def _parse_gpu_count(gres):
|
|
49
|
+
if not isinstance(gres, str):
|
|
50
|
+
return 0
|
|
51
|
+
gres = gres.strip()
|
|
52
|
+
if gres in ('(null)', ''):
|
|
53
|
+
return 0
|
|
54
|
+
|
|
55
|
+
n_gpu = 0
|
|
56
|
+
for token in gres.split(','):
|
|
57
|
+
token = token.strip()
|
|
58
|
+
if not token.lower().startswith('gpu'):
|
|
59
|
+
continue
|
|
60
|
+
token_core = token.split('(')[0]
|
|
61
|
+
parts = token_core.split(':')
|
|
62
|
+
count = None
|
|
63
|
+
|
|
64
|
+
if len(parts) >= 2 and re.fullmatch(r'\d+', parts[-1]):
|
|
65
|
+
count = int(parts[-1])
|
|
66
|
+
elif len(parts) >= 3 and re.fullmatch(r'\d+', parts[-2]):
|
|
67
|
+
count = int(parts[-2])
|
|
68
|
+
|
|
69
|
+
n_gpu += count if count is not None else 1
|
|
70
|
+
return n_gpu
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _parse_gpu_models(gres):
|
|
74
|
+
if not isinstance(gres, str):
|
|
51
75
|
return 'n/a'
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
76
|
+
gres = gres.strip()
|
|
77
|
+
if gres in ('(null)', ''):
|
|
78
|
+
return 'n/a'
|
|
79
|
+
|
|
80
|
+
models = []
|
|
81
|
+
for token in gres.split(','):
|
|
82
|
+
token = token.strip()
|
|
83
|
+
if not token.lower().startswith('gpu'):
|
|
84
|
+
continue
|
|
85
|
+
token_core = token.split('(')[0]
|
|
86
|
+
parts = token_core.split(':')
|
|
87
|
+
if len(parts) >= 3 and not re.fullmatch(r'\d+', parts[1]):
|
|
88
|
+
model = parts[1]
|
|
89
|
+
elif len(parts) == 2 and not re.fullmatch(r'\d+', parts[1]):
|
|
90
|
+
model = parts[1]
|
|
91
|
+
elif len(parts) == 2 and re.fullmatch(r'\d+', parts[1]):
|
|
92
|
+
model = 'gpu'
|
|
55
93
|
else:
|
|
56
|
-
|
|
57
|
-
|
|
94
|
+
model = 'n/a'
|
|
95
|
+
if model not in models:
|
|
96
|
+
models.append(model)
|
|
97
|
+
|
|
98
|
+
return ','.join(models) if models else 'n/a'
|
|
58
99
|
|
|
59
100
|
|
|
60
101
|
def _read_nodes(reservations=None, pending_res=None, return_raw=False):
|
|
61
102
|
sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c;%f"').read()), sep=';')
|
|
62
103
|
sinfo_df = _split_column(sinfo_df, 'NODELIST').drop_duplicates()
|
|
63
|
-
sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(
|
|
64
|
-
|
|
65
|
-
sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
|
|
104
|
+
sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_count)
|
|
105
|
+
sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_models)
|
|
66
106
|
sinfo_df['reserved'] = 'no'
|
|
67
107
|
if reservations is not None:
|
|
68
108
|
sinfo_df.loc[sinfo_df['NODELIST'].isin(reservations.Nodes.unique()), 'reserved'] = 'yes'
|
|
@@ -3,6 +3,8 @@ from . import slurm_list, slurm_viz, styles
|
|
|
3
3
|
|
|
4
4
|
def update_views(stdscr, instance, filter):
|
|
5
5
|
if instance.jobs is None or instance.inf is None:
|
|
6
|
+
instance.rens = "No data received from master yet.\nTry running with --debug or --diag."
|
|
7
|
+
instance.nocc = "Waiting for infrastructure snapshot..."
|
|
6
8
|
return
|
|
7
9
|
|
|
8
10
|
instance.rens = slurm_list.view_list(
|
|
@@ -11,5 +13,11 @@ def update_views(stdscr, instance, filter):
|
|
|
11
13
|
instance, 'job_id_type') else 'agg')
|
|
12
14
|
instance.timeme(f"- print job list")
|
|
13
15
|
|
|
14
|
-
instance.nocc = slurm_viz.view_viz(
|
|
16
|
+
instance.nocc = slurm_viz.view_viz(
|
|
17
|
+
instance.inf,
|
|
18
|
+
instance.jobs,
|
|
19
|
+
stylefn=styles.crsstyler,
|
|
20
|
+
mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu',
|
|
21
|
+
panel_width=instance.right_width if hasattr(instance, 'right_width') else None,
|
|
22
|
+
)
|
|
15
23
|
instance.timeme(f"- view nodeocc")
|
|
@@ -35,6 +35,11 @@ BUTTON_ACTIONS = {
|
|
|
35
35
|
'info': [ord('i')],
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
RIGHT_WIDTH_GPU = 33
|
|
39
|
+
RIGHT_WIDTH_INFO = 39
|
|
40
|
+
# Keep a single stable width across modes; use the widest one so info fields are not cut.
|
|
41
|
+
RIGHT_WIDTH_UNIFIED = max(RIGHT_WIDTH_GPU, RIGHT_WIDTH_INFO)
|
|
42
|
+
|
|
38
43
|
|
|
39
44
|
def is_file_writable_byall(file_path):
|
|
40
45
|
return os.stat(file_path).st_mode & stat.S_IWOTH
|
|
@@ -44,12 +49,49 @@ def is_file_readable_byall(file_path):
|
|
|
44
49
|
return os.stat(file_path).st_mode & stat.S_IROTH
|
|
45
50
|
|
|
46
51
|
|
|
52
|
+
def is_nodeocc_process(pid):
|
|
53
|
+
try:
|
|
54
|
+
cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
|
|
55
|
+
return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
|
|
56
|
+
except Exception:
|
|
57
|
+
return False
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def port_from_portfile(portfile_path):
|
|
61
|
+
try:
|
|
62
|
+
return int(Path(portfile_path).name.split('.')[0].split('master_')[1])
|
|
63
|
+
except Exception:
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def process_has_udp_port(pid, port):
|
|
68
|
+
if port is None:
|
|
69
|
+
return False
|
|
70
|
+
try:
|
|
71
|
+
proc = psutil.Process(int(pid))
|
|
72
|
+
for conn in proc.connections(kind='udp'):
|
|
73
|
+
if conn.laddr and conn.laddr.port == int(port):
|
|
74
|
+
return True
|
|
75
|
+
except Exception:
|
|
76
|
+
return False
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
|
|
47
80
|
def try_open_socket_as_slave(instance, force=False):
|
|
48
81
|
if len(instance.get_port_files()) == 0:
|
|
49
82
|
raise Exception("No master running")
|
|
50
83
|
|
|
51
|
-
|
|
52
|
-
|
|
84
|
+
active_ports = []
|
|
85
|
+
for f in instance.get_port_files():
|
|
86
|
+
pid = Path(f).read_text().strip()
|
|
87
|
+
port = port_from_portfile(f)
|
|
88
|
+
if pid and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
|
|
89
|
+
active_ports.append(port)
|
|
90
|
+
|
|
91
|
+
if len(active_ports) == 0:
|
|
92
|
+
raise Exception("No active master process found from port files")
|
|
93
|
+
|
|
94
|
+
cur_port = active_ports[0]
|
|
53
95
|
|
|
54
96
|
if hasattr(instance, 'port'):
|
|
55
97
|
# check if port file has same name
|
|
@@ -96,15 +138,20 @@ class Singleton:
|
|
|
96
138
|
if not os.path.exists(bp):
|
|
97
139
|
return True
|
|
98
140
|
for f in os.listdir(bp):
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
pid = Path(
|
|
102
|
-
|
|
103
|
-
if
|
|
104
|
-
|
|
141
|
+
fp = os.path.join(bp, f)
|
|
142
|
+
if 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(fp):
|
|
143
|
+
pid = Path(fp).read_text().strip()
|
|
144
|
+
port = port_from_portfile(fp)
|
|
145
|
+
if not pid:
|
|
146
|
+
os.remove(fp)
|
|
147
|
+
continue
|
|
148
|
+
if pid == str(os.getpid()):
|
|
149
|
+
os.remove(fp)
|
|
150
|
+
continue
|
|
151
|
+
if (not force) and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
|
|
152
|
+
self.log(f"Process {pid} is still running, keeping existing master")
|
|
105
153
|
return False
|
|
106
|
-
|
|
107
|
-
os.system(f"rm {os.path.join(bp, f)}")
|
|
154
|
+
os.remove(fp)
|
|
108
155
|
return True
|
|
109
156
|
|
|
110
157
|
def __init__(self, args=None):
|
|
@@ -115,6 +162,7 @@ class Singleton:
|
|
|
115
162
|
Singleton.__instance = self
|
|
116
163
|
|
|
117
164
|
self.args = args
|
|
165
|
+
self.basepath = self.args.basepath
|
|
118
166
|
self.setup_logging()
|
|
119
167
|
|
|
120
168
|
self.try_open_counter = 0
|
|
@@ -141,11 +189,13 @@ class Singleton:
|
|
|
141
189
|
self.k = -1
|
|
142
190
|
|
|
143
191
|
# check if user group is 'student' or 'tesisti'
|
|
144
|
-
|
|
192
|
+
try:
|
|
193
|
+
user_group = grp.getgrgid(os.getgid()).gr_name
|
|
194
|
+
except KeyError:
|
|
195
|
+
user_group = ''
|
|
196
|
+
self.err(f"Could not resolve gid {os.getgid()} via grp.getgrgid")
|
|
145
197
|
self.cur_partition = 'stud' if user_group in ['studenti', 'tesisti'] else 'prod'
|
|
146
198
|
|
|
147
|
-
self.basepath = self.args.basepath
|
|
148
|
-
|
|
149
199
|
self.is_master = not self.check_existing_master_running()
|
|
150
200
|
|
|
151
201
|
if self.is_master or args.override:
|
|
@@ -158,9 +208,16 @@ class Singleton:
|
|
|
158
208
|
if len(portfiles) > 0:
|
|
159
209
|
for f in portfiles:
|
|
160
210
|
# check if process is still running
|
|
161
|
-
pid = Path(f).read_text()
|
|
162
|
-
if
|
|
163
|
-
|
|
211
|
+
pid = Path(f).read_text().strip()
|
|
212
|
+
if not pid:
|
|
213
|
+
continue
|
|
214
|
+
try:
|
|
215
|
+
pid_int = int(pid)
|
|
216
|
+
except ValueError:
|
|
217
|
+
continue
|
|
218
|
+
port = port_from_portfile(f)
|
|
219
|
+
if pid_int != os.getpid() and is_nodeocc_process(pid_int) and process_has_udp_port(pid_int, port):
|
|
220
|
+
return True
|
|
164
221
|
return False
|
|
165
222
|
|
|
166
223
|
def get_port_files(self):
|
|
@@ -182,11 +239,6 @@ class Singleton:
|
|
|
182
239
|
# get pid of current process
|
|
183
240
|
self.pid = os.getpid()
|
|
184
241
|
|
|
185
|
-
# clean up old port files
|
|
186
|
-
kill_ok = self.clean_port_files(force=force)
|
|
187
|
-
if not kill_ok:
|
|
188
|
-
return None
|
|
189
|
-
|
|
190
242
|
try:
|
|
191
243
|
# create file to store port with 666 permissions to file
|
|
192
244
|
# file is in folder basepath/nodename/master_{port}.portfile
|
|
@@ -211,10 +263,15 @@ class Singleton:
|
|
|
211
263
|
# check if file exists
|
|
212
264
|
if filepath.exists():
|
|
213
265
|
# delete file
|
|
214
|
-
os.
|
|
266
|
+
os.remove(filepath)
|
|
215
267
|
return None
|
|
216
268
|
|
|
217
269
|
def create_socket_as_master(self, force=False):
|
|
270
|
+
# clean up old port files before creating the socket to avoid inconsistent state
|
|
271
|
+
kill_ok = self.clean_port_files(force=force)
|
|
272
|
+
if not kill_ok:
|
|
273
|
+
return None
|
|
274
|
+
|
|
218
275
|
# create udp socket for broadcasting
|
|
219
276
|
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
|
|
220
277
|
|
|
@@ -248,8 +305,6 @@ class Singleton:
|
|
|
248
305
|
self.sock.settimeout(6.5)
|
|
249
306
|
self.log(f"Socket opened on port {self.port}")
|
|
250
307
|
|
|
251
|
-
self.update_port_file(force=force)
|
|
252
|
-
|
|
253
308
|
self.try_open_counter = 0
|
|
254
309
|
return self.port
|
|
255
310
|
except Exception as e:
|
|
@@ -426,7 +481,7 @@ def handle_keys(stdscr, instance):
|
|
|
426
481
|
instance.sort_by_prio = not instance.sort_by_prio
|
|
427
482
|
elif k in BUTTON_ACTIONS['res_view_mode']:
|
|
428
483
|
instance.view_mode = {"gpu": "ram", "ram": "cpu", "cpu": "gpu", "info": "gpu"}[instance.view_mode]
|
|
429
|
-
instance.right_width =
|
|
484
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
430
485
|
stdscr.clear()
|
|
431
486
|
elif k in BUTTON_ACTIONS['job_id_type']:
|
|
432
487
|
instance.job_id_type = "true" if instance.job_id_type == "agg" else "agg"
|
|
@@ -439,7 +494,7 @@ def handle_keys(stdscr, instance):
|
|
|
439
494
|
instance.show_prio = not instance.show_prio
|
|
440
495
|
elif k in BUTTON_ACTIONS['info']:
|
|
441
496
|
instance.view_mode = 'info' if instance.view_mode != 'info' else 'gpu'
|
|
442
|
-
instance.right_width =
|
|
497
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
443
498
|
stdscr.clear()
|
|
444
499
|
elif k in BUTTON_ACTIONS['up_tab']:
|
|
445
500
|
# get screen size
|
|
@@ -463,7 +518,7 @@ def update_screen(stdscr, instance):
|
|
|
463
518
|
if instance.k in BUTTON_ACTIONS['refresh']:
|
|
464
519
|
stdscr.clear()
|
|
465
520
|
|
|
466
|
-
totsize = 106
|
|
521
|
+
totsize = 106 + max(0, RIGHT_WIDTH_UNIFIED - RIGHT_WIDTH_GPU)
|
|
467
522
|
if instance.show_account:
|
|
468
523
|
totsize += 10
|
|
469
524
|
if instance.show_prio:
|
|
@@ -556,8 +611,14 @@ def update_screen(stdscr, instance):
|
|
|
556
611
|
# get slurm user partition
|
|
557
612
|
# stdscr.addstr(lines - 1, xoffset + 62 + 2, f'(Avg time {instance.avg_wait_time})', curses.color_pair(2))
|
|
558
613
|
|
|
559
|
-
|
|
560
|
-
|
|
614
|
+
if instance.updated:
|
|
615
|
+
signature = instance.signature
|
|
616
|
+
sig_style = curses.color_pair(2)
|
|
617
|
+
else:
|
|
618
|
+
signature = f'{instance.signature} (new {instance.newest_version})'
|
|
619
|
+
sig_style = curses.color_pair(4)
|
|
620
|
+
|
|
621
|
+
stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, sig_style)
|
|
561
622
|
|
|
562
623
|
stdscr.refresh()
|
|
563
624
|
curses.doupdate()
|
|
@@ -662,7 +723,7 @@ async def main(stdscr):
|
|
|
662
723
|
# status
|
|
663
724
|
|
|
664
725
|
instance.a_filter = 0
|
|
665
|
-
instance.right_width =
|
|
726
|
+
instance.right_width = RIGHT_WIDTH_UNIFIED
|
|
666
727
|
|
|
667
728
|
stdscr.clear()
|
|
668
729
|
|
|
@@ -44,7 +44,7 @@ def _joblet_format(instance, job, width=74, jobid_type='agg'):
|
|
|
44
44
|
mem = round(joblet.mem / 1024)
|
|
45
45
|
joblet_repr += _format_to(f'{mem}G', 4, 'right')
|
|
46
46
|
joblet_repr += ' '
|
|
47
|
-
if instance.view_mode in ('gpu', 'ram'):
|
|
47
|
+
if instance.view_mode in ('gpu', 'ram', 'info'):
|
|
48
48
|
joblet_repr += _format_to((f'{joblet.n_gpus}gp' if joblet.n_gpus > 0 else " - "), 3, 'left')
|
|
49
49
|
elif instance.view_mode == 'cpu':
|
|
50
50
|
joblet_repr += _format_to((f'{joblet.cpus}cp' if joblet.cpus > 0 else " - "), 3, 'left')
|
|
@@ -101,6 +101,20 @@ cpu_paused = '▚'
|
|
|
101
101
|
cpu_down = '⨯'
|
|
102
102
|
|
|
103
103
|
GLOBAL_NODES = None
|
|
104
|
+
BASE_PANEL_WIDTH = 33
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _center_panel_content(text, panel_width, base_width=BASE_PANEL_WIDTH):
|
|
108
|
+
if panel_width is None or panel_width <= base_width:
|
|
109
|
+
return text
|
|
110
|
+
|
|
111
|
+
pad = max(0, (panel_width - base_width) // 2)
|
|
112
|
+
if pad == 0:
|
|
113
|
+
return text
|
|
114
|
+
|
|
115
|
+
lines = text.split('\n')
|
|
116
|
+
lines = [(' ' * pad + line) if line else line for line in lines]
|
|
117
|
+
return '\n'.join(lines)
|
|
104
118
|
|
|
105
119
|
|
|
106
120
|
def get_ram_block(megs):
|
|
@@ -138,15 +152,15 @@ def get_node_status(node, jobs):
|
|
|
138
152
|
return None # white?
|
|
139
153
|
|
|
140
154
|
|
|
141
|
-
def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu'):
|
|
155
|
+
def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu', panel_width=None):
|
|
142
156
|
if mode == 'gpu':
|
|
143
|
-
return view_viz_gpu(infrastructure, jobs, work, stylefn, current_user)
|
|
157
|
+
return _center_panel_content(view_viz_gpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
144
158
|
elif mode == 'ram':
|
|
145
|
-
return view_viz_ram(infrastructure, jobs, work, stylefn, current_user)
|
|
159
|
+
return _center_panel_content(view_viz_ram(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
146
160
|
elif mode == 'cpu':
|
|
147
|
-
return view_viz_cpu(infrastructure, jobs, work, stylefn, current_user)
|
|
161
|
+
return _center_panel_content(view_viz_cpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
|
|
148
162
|
elif mode == 'info':
|
|
149
|
-
return view_viz_info(infrastructure, jobs, work, stylefn, current_user)
|
|
163
|
+
return view_viz_info(infrastructure, jobs, work, stylefn, current_user, panel_width)
|
|
150
164
|
|
|
151
165
|
|
|
152
166
|
def view_viz_ram(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
|
|
@@ -519,7 +533,7 @@ def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user
|
|
|
519
533
|
return RetScope.return_string
|
|
520
534
|
|
|
521
535
|
|
|
522
|
-
def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
|
|
536
|
+
def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, panel_width=None):
|
|
523
537
|
# this is for hot reload
|
|
524
538
|
if not work:
|
|
525
539
|
return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
|
|
@@ -555,6 +569,7 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
|
|
|
555
569
|
# max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
|
|
556
570
|
feat_str = stylefn("GREEN", "FEATURES")
|
|
557
571
|
RetScope.return_string += f'{"":{gpu_name_chars}} GPU {feat_str}\n'
|
|
572
|
+
|
|
558
573
|
for i, n in GLOBAL_NODES.iterrows():
|
|
559
574
|
gpu_model = n.m_gpu # if n.m_gpu is not None else 'N/A'
|
|
560
575
|
# gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|