nodeocc 1.0.24__tar.gz → 1.0.26__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {nodeocc-1.0.24 → nodeocc-1.0.26}/PKG-INFO +2 -2
  2. {nodeocc-1.0.24 → nodeocc-1.0.26}/pyproject.toml +1 -1
  3. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/controller/controller.py +132 -68
  4. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/PKG-INFO +2 -2
  5. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/readers/slurmreader.py +51 -11
  6. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/__init__.py +9 -1
  7. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/curses_multiwindow.py +91 -30
  8. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/slurm_list.py +1 -1
  9. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/slurm_viz.py +21 -6
  10. {nodeocc-1.0.24 → nodeocc-1.0.26}/README.md +0 -0
  11. {nodeocc-1.0.24 → nodeocc-1.0.26}/setup.cfg +0 -0
  12. {nodeocc-1.0.24 → nodeocc-1.0.26}/setup.py +0 -0
  13. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/__init__.py +0 -0
  14. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/model/infrastructure.py +0 -0
  15. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/model/job.py +0 -0
  16. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/SOURCES.txt +0 -0
  17. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/dependency_links.txt +0 -0
  18. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/entry_points.txt +0 -0
  19. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/requires.txt +0 -0
  20. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/nodeocc.egg-info/top_level.txt +0 -0
  21. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/readers/compute_wait_time.py +0 -0
  22. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/styles.py +0 -0
  23. {nodeocc-1.0.24 → nodeocc-1.0.26}/src/view/utils.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: nodeocc
3
- Version: 1.0.24
3
+ Version: 1.0.26
4
4
  Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
5
5
  Author: Matteo Boschini
6
6
  Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "nodeocc"
7
- version = "1.0.24"
7
+ version = "1.0.26"
8
8
  authors = [{ name = "Matteo Boschini" }]
9
9
  maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
10
10
  description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"
@@ -8,7 +8,7 @@ import numpy as np
8
8
 
9
9
  conf_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
10
10
  print(conf_path)
11
- sys.path.append(conf_path)
11
+ sys.path.insert(0, conf_path)
12
12
  from curses import wrapper
13
13
  from view.curses_multiwindow import main, Singleton, try_open_socket_as_slave
14
14
 
@@ -23,6 +23,9 @@ import time
23
23
  import json
24
24
  import argparse
25
25
  import traceback
26
+ import socket
27
+ import re
28
+ from urllib.request import urlopen
26
29
  from model.infrastructure import Infrastructure
27
30
  from model.job import Job
28
31
  import asyncio
@@ -30,36 +33,28 @@ import importlib.metadata
30
33
 
31
34
  program_name = 'nodeocc'
32
35
  version_number = importlib.metadata.version(program_name)
33
- updated = False
34
36
 
35
- BBLUE = '\033[1;34m'
36
- END = '\033[0m'
37
- try:
38
- url = "https://pypi.python.org/pypi/nodeocc/json"
39
- import requests
40
- newest_version = json.loads(requests.get(url).text)['info']['version']
41
- print(BBLUE, end='')
42
- if int(newest_version.split('.')[0]) > int(version_number.split('.')[0]):
43
- print(f"New major version available: {newest_version} (current: {version_number})")
44
- print(f"Please update with 'pip install {program_name} --upgrade'")
45
- time.sleep(7)
46
- elif int(newest_version.split('.')[1]) > int(version_number.split('.')[1]):
47
- print(f"New minor version available: {newest_version} (current: {version_number})")
48
- print(f"Please update with 'pip install {program_name} --upgrade'")
49
- time.sleep(7)
50
- elif int(newest_version.split('.')[2]) > int(version_number.split('.')[2]):
51
- print(f"New patch version available: {newest_version} (current: {version_number})")
52
- print(f"Please update with 'pip install {program_name} --upgrade'")
53
- time.sleep(7)
54
- else:
55
- print(f"Up to date: {version_number}")
56
- updated = True
57
- time.sleep(3)
58
- print(END, end='')
59
- except Exception as e:
60
- print(f"{BBLUE}Could not get newest version from pip, check if {program_name} is up to date{END}")
61
- newest_version = '?.?.?'
62
- time.sleep(5)
37
+
38
+ def _version_tuple(v):
39
+ return tuple(int(x) for x in re.findall(r'\d+', str(v)))
40
+
41
+
42
+ def check_latest_version(cur_version, timeout_sec=1.2):
43
+ """
44
+ Fast, non-blocking-ish version check:
45
+ - short timeout
46
+ - no startup sleeps/pauses
47
+ - graceful fallback to current version on errors
48
+ """
49
+ try:
50
+ url = f"https://pypi.python.org/pypi/{program_name}/json"
51
+ with urlopen(url, timeout=timeout_sec) as response:
52
+ payload = json.loads(response.read().decode('utf-8'))
53
+ newest = payload['info']['version']
54
+ is_updated = _version_tuple(cur_version) >= _version_tuple(newest)
55
+ return newest, is_updated
56
+ except Exception:
57
+ return cur_version, True
63
58
 
64
59
  BEGIN_DELIM = "!{$"
65
60
  END_DELIM_ENCODED = "!}$".encode('utf-8')
@@ -74,11 +69,18 @@ parser.add_argument('--debug', action='store_true', help='Enable logging')
74
69
  parser.add_argument('--override', action='store_true', help='Enable logging')
75
70
  parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
76
71
  parser.add_argument('--basepath', type=str, default='/tmp', help='Base path for nodeocc. Must be readable by all users')
72
+ parser.add_argument('--diag', action='store_true', help='Run one-shot diagnostics and exit')
77
73
  args = parser.parse_args()
78
74
 
79
75
  # check basepath is readable
80
76
  assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
81
77
 
78
+ # Keep startup snappy for daemon/diagnostics; check updates only in interactive mode.
79
+ if args.master_only or args.diag:
80
+ newest_version, updated = version_number, True
81
+ else:
82
+ newest_version, updated = check_latest_version(version_number)
83
+
82
84
  # export args
83
85
  orig_instance = Singleton.getInstance(args)
84
86
 
@@ -177,49 +179,61 @@ async def get_data_slave(instance):
177
179
  # listen for data from master
178
180
  # instance.sock.settimeout(6.5)
179
181
  data, _ = instance.sock.recvfrom(MAX_BUF)
180
- decoded_data = data.decode('utf-8')
181
- if BEGIN_DELIM in decoded_data:
182
- n_msgs_remaining = int(decoded_data.split(BEGIN_DELIM)[0])
183
- # msg_len += len(str(msg_len)) message len is deprecated and used for n_msgs_remaining
184
- instance.timeme(f"received first message of {n_msgs_remaining} (total len {len(data)} bytes)")
185
-
186
- inf, jobs, avg_wait_time, decoded_msg = decode_msg_slave(data)
187
- instance.timeme(f"- receive")
188
- orig_timestamp = float(decoded_msg['ts'])
189
-
190
- total_msg_n = n_msgs_remaining
191
- msg_store = {i: [] for i in range(total_msg_n)}
192
- while n_msgs_remaining > 0:
193
- data, _ = instance.sock.recvfrom(MAX_BUF)
194
- back_msg_idx = int(data.decode('utf-8').split(EXTRA_MSG_BEGIN_DELIM)[0])
195
- msg_idx = total_msg_n - back_msg_idx - 1
196
- _, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
197
-
198
- # check if timestamp is the same
199
- if float(new_ts['ts']) != orig_timestamp:
200
- instance.timeme(f"Timestamp mismatch, skipping")
201
- return inf, jobs, avg_wait_time # return what we have
202
-
203
- msg_store[msg_idx] += new_jobs
204
-
205
- n_msgs_remaining -= 1
206
- instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
207
-
208
- # sort and merge
209
- for msg_idx in range(total_msg_n):
210
- if len(msg_store[msg_idx]) == 0:
211
- instance.log(f"Empty message {msg_idx}")
212
- jobs += msg_store[msg_idx]
213
- else:
214
- instance.timeme(f"- no data")
215
- return None, None, None, None
182
+ decoded_data = data.decode('utf-8', errors='replace')
183
+ if BEGIN_DELIM not in decoded_data:
184
+ if EXTRA_MSG_BEGIN_DELIM in decoded_data:
185
+ instance.log("Received extra packet before main packet, discarding")
186
+ else:
187
+ instance.log("Received packet without known delimiters, discarding")
188
+ return None, None, None
189
+
190
+ n_msgs_remaining = int(decoded_data.split(BEGIN_DELIM)[0])
191
+ # msg_len += len(str(msg_len)) message len is deprecated and used for n_msgs_remaining
192
+ instance.timeme(f"received first message of {n_msgs_remaining} (total len {len(data)} bytes)")
193
+
194
+ inf, jobs, avg_wait_time, decoded_msg = decode_msg_slave(data)
195
+ instance.timeme(f"- receive")
196
+ orig_timestamp = float(decoded_msg['ts'])
197
+
198
+ total_msg_n = n_msgs_remaining
199
+ msg_store = {i: [] for i in range(total_msg_n)}
200
+ while n_msgs_remaining > 0:
201
+ data, _ = instance.sock.recvfrom(MAX_BUF)
202
+ decoded_extra = data.decode('utf-8', errors='replace')
203
+ if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
204
+ instance.log("Received non-extra packet while waiting for extra packets, skipping")
205
+ continue
206
+
207
+ back_msg_idx = int(decoded_extra.split(EXTRA_MSG_BEGIN_DELIM)[0])
208
+ msg_idx = total_msg_n - back_msg_idx - 1
209
+ if msg_idx < 0 or msg_idx >= total_msg_n:
210
+ instance.log(f"Invalid extra packet index: {msg_idx}, skipping")
211
+ continue
212
+
213
+ _, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
214
+
215
+ # check if timestamp is the same
216
+ if float(new_ts['ts']) != orig_timestamp:
217
+ instance.timeme(f"Timestamp mismatch, skipping")
218
+ return inf, jobs, avg_wait_time # return what we have
219
+
220
+ msg_store[msg_idx] += new_jobs
221
+
222
+ n_msgs_remaining -= 1
223
+ instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
224
+
225
+ # sort and merge
226
+ for msg_idx in range(total_msg_n):
227
+ if len(msg_store[msg_idx]) == 0:
228
+ instance.log(f"Empty message {msg_idx}")
229
+ jobs += msg_store[msg_idx]
216
230
 
217
231
  n_timeout = 0
218
232
 
219
- except BlockingIOError as e:
233
+ except BlockingIOError:
220
234
  pass
221
235
 
222
- except TimeoutError as e:
236
+ except TimeoutError:
223
237
  n_timeout += 1
224
238
  instance.log(f"TIMEOUT {n_timeout}/{TIMEOUT_MAX}")
225
239
  if n_timeout > TIMEOUT_MAX:
@@ -227,6 +241,10 @@ async def get_data_slave(instance):
227
241
 
228
242
  time.sleep(random.randint(100, 10000) / 1000)
229
243
 
244
+ except Exception as e:
245
+ instance.err(f"Exception while receiving data from master: {e}")
246
+ instance.err(traceback.format_exc())
247
+
230
248
  return inf, jobs, avg_wait_time
231
249
 
232
250
 
@@ -274,7 +292,53 @@ def display_main(stdscr):
274
292
  return asyncio.run(main(stdscr))
275
293
 
276
294
 
295
+ def run_diagnostics(instance: Singleton):
296
+ hostname = socket.getfqdn().split('.')[0]
297
+ print(f"Diagnostics for nodeocc on host: {hostname}")
298
+ print(f"Base path: {instance.basepath}")
299
+ print(f"Role detected: {'master' if instance.is_master else 'slave'}")
300
+ print("")
301
+
302
+ try:
303
+ port_files = instance.get_port_files()
304
+ print(f"Visible port files: {len(port_files)}")
305
+ for pf in port_files:
306
+ try:
307
+ pid = Path(pf).read_text().strip()
308
+ except Exception:
309
+ pid = "?"
310
+ print(f" - {pf} (pid={pid})")
311
+ except Exception as e:
312
+ print(f"Could not list port files: {e}")
313
+
314
+ print("")
315
+
316
+ if instance.is_master:
317
+ print("Running one master update cycle...")
318
+ try:
319
+ inf, jobs = update_data_master(instance)
320
+ print(f"Master update OK. Nodes={len(inf.nodes)}, Jobs={len(jobs)}, Port={instance.port}")
321
+ except Exception as e:
322
+ print(f"Master update FAILED: {e}")
323
+ print(traceback.format_exc())
324
+ else:
325
+ print("Waiting for one packet from master...")
326
+ try:
327
+ inf, jobs, _ = asyncio.run(get_data_slave(instance))
328
+ if inf is None or jobs is None:
329
+ print("No valid payload received from master.")
330
+ else:
331
+ print(f"Slave receive OK. Jobs in snapshot={len(jobs)}")
332
+ except Exception as e:
333
+ print(f"Slave receive FAILED: {e}")
334
+ print(traceback.format_exc())
335
+
336
+
277
337
  def _main():
338
+ if args.diag:
339
+ run_diagnostics(orig_instance)
340
+ return
341
+
278
342
  if args.master_only:
279
343
  assert orig_instance.is_master, "Instance is not `master`, run with `force_override` to override"
280
344
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: nodeocc
3
- Version: 1.0.24
3
+ Version: 1.0.26
4
4
  Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
5
5
  Author: Matteo Boschini
6
6
  Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -45,24 +45,64 @@ def _node_from_sinfo(x):
45
45
  return n
46
46
 
47
47
 
48
- def _parse_gpu_models(line):
49
- line = line[0].split(',')
50
- if line[0] == '(null)':
48
+ def _parse_gpu_count(gres):
49
+ if not isinstance(gres, str):
50
+ return 0
51
+ gres = gres.strip()
52
+ if gres in ('(null)', ''):
53
+ return 0
54
+
55
+ n_gpu = 0
56
+ for token in gres.split(','):
57
+ token = token.strip()
58
+ if not token.lower().startswith('gpu'):
59
+ continue
60
+ token_core = token.split('(')[0]
61
+ parts = token_core.split(':')
62
+ count = None
63
+
64
+ if len(parts) >= 2 and re.fullmatch(r'\d+', parts[-1]):
65
+ count = int(parts[-1])
66
+ elif len(parts) >= 3 and re.fullmatch(r'\d+', parts[-2]):
67
+ count = int(parts[-2])
68
+
69
+ n_gpu += count if count is not None else 1
70
+ return n_gpu
71
+
72
+
73
+ def _parse_gpu_models(gres):
74
+ if not isinstance(gres, str):
51
75
  return 'n/a'
52
- else:
53
- if len(line) == 1:
54
- return line[0].split(':')[1]
76
+ gres = gres.strip()
77
+ if gres in ('(null)', ''):
78
+ return 'n/a'
79
+
80
+ models = []
81
+ for token in gres.split(','):
82
+ token = token.strip()
83
+ if not token.lower().startswith('gpu'):
84
+ continue
85
+ token_core = token.split('(')[0]
86
+ parts = token_core.split(':')
87
+ if len(parts) >= 3 and not re.fullmatch(r'\d+', parts[1]):
88
+ model = parts[1]
89
+ elif len(parts) == 2 and not re.fullmatch(r'\d+', parts[1]):
90
+ model = parts[1]
91
+ elif len(parts) == 2 and re.fullmatch(r'\d+', parts[1]):
92
+ model = 'gpu'
55
93
  else:
56
- # return line[0].split(':')[1] + ',' + line[1].split(':')[1]
57
- return ','.join([x.split(':')[1] for x in line])
94
+ model = 'n/a'
95
+ if model not in models:
96
+ models.append(model)
97
+
98
+ return ','.join(models) if models else 'n/a'
58
99
 
59
100
 
60
101
  def _read_nodes(reservations=None, pending_res=None, return_raw=False):
61
102
  sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c;%f"').read()), sep=';')
62
103
  sinfo_df = _split_column(sinfo_df, 'NODELIST').drop_duplicates()
63
- sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(lambda x: 0 if x.split('(')[0].split(':')[0] in ('(null)', '') else int(x.split('(')[0].split(':')[-1]))
64
- # sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(':').apply(lambda x: 'n/a' if x[0] in ('(null)', '') else x[1])
65
- sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
104
+ sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_count)
105
+ sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_models)
66
106
  sinfo_df['reserved'] = 'no'
67
107
  if reservations is not None:
68
108
  sinfo_df.loc[sinfo_df['NODELIST'].isin(reservations.Nodes.unique()), 'reserved'] = 'yes'
@@ -3,6 +3,8 @@ from . import slurm_list, slurm_viz, styles
3
3
 
4
4
  def update_views(stdscr, instance, filter):
5
5
  if instance.jobs is None or instance.inf is None:
6
+ instance.rens = "No data received from master yet.\nTry running with --debug or --diag."
7
+ instance.nocc = "Waiting for infrastructure snapshot..."
6
8
  return
7
9
 
8
10
  instance.rens = slurm_list.view_list(
@@ -11,5 +13,11 @@ def update_views(stdscr, instance, filter):
11
13
  instance, 'job_id_type') else 'agg')
12
14
  instance.timeme(f"- print job list")
13
15
 
14
- instance.nocc = slurm_viz.view_viz(instance.inf, instance.jobs, stylefn=styles.crsstyler, mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu')
16
+ instance.nocc = slurm_viz.view_viz(
17
+ instance.inf,
18
+ instance.jobs,
19
+ stylefn=styles.crsstyler,
20
+ mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu',
21
+ panel_width=instance.right_width if hasattr(instance, 'right_width') else None,
22
+ )
15
23
  instance.timeme(f"- view nodeocc")
@@ -35,6 +35,11 @@ BUTTON_ACTIONS = {
35
35
  'info': [ord('i')],
36
36
  }
37
37
 
38
+ RIGHT_WIDTH_GPU = 33
39
+ RIGHT_WIDTH_INFO = 39
40
+ # Keep a single stable width across modes; use the widest one so info fields are not cut.
41
+ RIGHT_WIDTH_UNIFIED = max(RIGHT_WIDTH_GPU, RIGHT_WIDTH_INFO)
42
+
38
43
 
39
44
  def is_file_writable_byall(file_path):
40
45
  return os.stat(file_path).st_mode & stat.S_IWOTH
@@ -44,12 +49,49 @@ def is_file_readable_byall(file_path):
44
49
  return os.stat(file_path).st_mode & stat.S_IROTH
45
50
 
46
51
 
52
+ def is_nodeocc_process(pid):
53
+ try:
54
+ cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
55
+ return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
56
+ except Exception:
57
+ return False
58
+
59
+
60
+ def port_from_portfile(portfile_path):
61
+ try:
62
+ return int(Path(portfile_path).name.split('.')[0].split('master_')[1])
63
+ except Exception:
64
+ return None
65
+
66
+
67
+ def process_has_udp_port(pid, port):
68
+ if port is None:
69
+ return False
70
+ try:
71
+ proc = psutil.Process(int(pid))
72
+ for conn in proc.connections(kind='udp'):
73
+ if conn.laddr and conn.laddr.port == int(port):
74
+ return True
75
+ except Exception:
76
+ return False
77
+ return False
78
+
79
+
47
80
  def try_open_socket_as_slave(instance, force=False):
48
81
  if len(instance.get_port_files()) == 0:
49
82
  raise Exception("No master running")
50
83
 
51
- cur_port = [int(f.split('.')[0].split('master_')[1]) for f in instance.get_port_files()
52
- if (pid := Path(f).read_text()) and psutil.pid_exists(int(pid))][0]
84
+ active_ports = []
85
+ for f in instance.get_port_files():
86
+ pid = Path(f).read_text().strip()
87
+ port = port_from_portfile(f)
88
+ if pid and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
89
+ active_ports.append(port)
90
+
91
+ if len(active_ports) == 0:
92
+ raise Exception("No active master process found from port files")
93
+
94
+ cur_port = active_ports[0]
53
95
 
54
96
  if hasattr(instance, 'port'):
55
97
  # check if port file has same name
@@ -96,15 +138,20 @@ class Singleton:
96
138
  if not os.path.exists(bp):
97
139
  return True
98
140
  for f in os.listdir(bp):
99
- if 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(os.path.join(bp, f)):
100
- # read pid from file
101
- pid = Path(os.path.join(bp, f)).read_text()
102
- # assert no process is running
103
- if psutil.pid_exists(int(pid)) and not force:
104
- self.log(f"Process {pid} is still running, killing self")
141
+ fp = os.path.join(bp, f)
142
+ if 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(fp):
143
+ pid = Path(fp).read_text().strip()
144
+ port = port_from_portfile(fp)
145
+ if not pid:
146
+ os.remove(fp)
147
+ continue
148
+ if pid == str(os.getpid()):
149
+ os.remove(fp)
150
+ continue
151
+ if (not force) and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
152
+ self.log(f"Process {pid} is still running, keeping existing master")
105
153
  return False
106
-
107
- os.system(f"rm {os.path.join(bp, f)}")
154
+ os.remove(fp)
108
155
  return True
109
156
 
110
157
  def __init__(self, args=None):
@@ -115,6 +162,7 @@ class Singleton:
115
162
  Singleton.__instance = self
116
163
 
117
164
  self.args = args
165
+ self.basepath = self.args.basepath
118
166
  self.setup_logging()
119
167
 
120
168
  self.try_open_counter = 0
@@ -141,11 +189,13 @@ class Singleton:
141
189
  self.k = -1
142
190
 
143
191
  # check if user group is 'student' or 'tesisti'
144
- user_group = grp.getgrgid(os.getgid()).gr_name
192
+ try:
193
+ user_group = grp.getgrgid(os.getgid()).gr_name
194
+ except KeyError:
195
+ user_group = ''
196
+ self.err(f"Could not resolve gid {os.getgid()} via grp.getgrgid")
145
197
  self.cur_partition = 'stud' if user_group in ['studenti', 'tesisti'] else 'prod'
146
198
 
147
- self.basepath = self.args.basepath
148
-
149
199
  self.is_master = not self.check_existing_master_running()
150
200
 
151
201
  if self.is_master or args.override:
@@ -158,9 +208,16 @@ class Singleton:
158
208
  if len(portfiles) > 0:
159
209
  for f in portfiles:
160
210
  # check if process is still running
161
- pid = Path(f).read_text()
162
- if pid and int(pid) != os.getpid():
163
- return psutil.pid_exists(int(pid))
211
+ pid = Path(f).read_text().strip()
212
+ if not pid:
213
+ continue
214
+ try:
215
+ pid_int = int(pid)
216
+ except ValueError:
217
+ continue
218
+ port = port_from_portfile(f)
219
+ if pid_int != os.getpid() and is_nodeocc_process(pid_int) and process_has_udp_port(pid_int, port):
220
+ return True
164
221
  return False
165
222
 
166
223
  def get_port_files(self):
@@ -182,11 +239,6 @@ class Singleton:
182
239
  # get pid of current process
183
240
  self.pid = os.getpid()
184
241
 
185
- # clean up old port files
186
- kill_ok = self.clean_port_files(force=force)
187
- if not kill_ok:
188
- return None
189
-
190
242
  try:
191
243
  # create file to store port with 666 permissions to file
192
244
  # file is in folder basepath/nodename/master_{port}.portfile
@@ -211,10 +263,15 @@ class Singleton:
211
263
  # check if file exists
212
264
  if filepath.exists():
213
265
  # delete file
214
- os.system(f"rm {filepath}")
266
+ os.remove(filepath)
215
267
  return None
216
268
 
217
269
  def create_socket_as_master(self, force=False):
270
+ # clean up old port files before creating the socket to avoid inconsistent state
271
+ kill_ok = self.clean_port_files(force=force)
272
+ if not kill_ok:
273
+ return None
274
+
218
275
  # create udp socket for broadcasting
219
276
  self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
220
277
 
@@ -248,8 +305,6 @@ class Singleton:
248
305
  self.sock.settimeout(6.5)
249
306
  self.log(f"Socket opened on port {self.port}")
250
307
 
251
- self.update_port_file(force=force)
252
-
253
308
  self.try_open_counter = 0
254
309
  return self.port
255
310
  except Exception as e:
@@ -426,7 +481,7 @@ def handle_keys(stdscr, instance):
426
481
  instance.sort_by_prio = not instance.sort_by_prio
427
482
  elif k in BUTTON_ACTIONS['res_view_mode']:
428
483
  instance.view_mode = {"gpu": "ram", "ram": "cpu", "cpu": "gpu", "info": "gpu"}[instance.view_mode]
429
- instance.right_width = 33 if instance.view_mode != 'info' else 39
484
+ instance.right_width = RIGHT_WIDTH_UNIFIED
430
485
  stdscr.clear()
431
486
  elif k in BUTTON_ACTIONS['job_id_type']:
432
487
  instance.job_id_type = "true" if instance.job_id_type == "agg" else "agg"
@@ -439,7 +494,7 @@ def handle_keys(stdscr, instance):
439
494
  instance.show_prio = not instance.show_prio
440
495
  elif k in BUTTON_ACTIONS['info']:
441
496
  instance.view_mode = 'info' if instance.view_mode != 'info' else 'gpu'
442
- instance.right_width = 33 if instance.view_mode != 'info' else 39
497
+ instance.right_width = RIGHT_WIDTH_UNIFIED
443
498
  stdscr.clear()
444
499
  elif k in BUTTON_ACTIONS['up_tab']:
445
500
  # get screen size
@@ -463,7 +518,7 @@ def update_screen(stdscr, instance):
463
518
  if instance.k in BUTTON_ACTIONS['refresh']:
464
519
  stdscr.clear()
465
520
 
466
- totsize = 106
521
+ totsize = 106 + max(0, RIGHT_WIDTH_UNIFIED - RIGHT_WIDTH_GPU)
467
522
  if instance.show_account:
468
523
  totsize += 10
469
524
  if instance.show_prio:
@@ -556,8 +611,14 @@ def update_screen(stdscr, instance):
556
611
  # get slurm user partition
557
612
  # stdscr.addstr(lines - 1, xoffset + 62 + 2, f'(Avg time {instance.avg_wait_time})', curses.color_pair(2))
558
613
 
559
- signature = instance.signature if instance.updated else f'{instance.version} -> {instance.newest_version}'
560
- stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, curses.color_pair(2) if instance.updated else curses.color_pair(3))
614
+ if instance.updated:
615
+ signature = instance.signature
616
+ sig_style = curses.color_pair(2)
617
+ else:
618
+ signature = f'{instance.signature} (new {instance.newest_version})'
619
+ sig_style = curses.color_pair(4)
620
+
621
+ stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, sig_style)
561
622
 
562
623
  stdscr.refresh()
563
624
  curses.doupdate()
@@ -662,7 +723,7 @@ async def main(stdscr):
662
723
  # status
663
724
 
664
725
  instance.a_filter = 0
665
- instance.right_width = 33
726
+ instance.right_width = RIGHT_WIDTH_UNIFIED
666
727
 
667
728
  stdscr.clear()
668
729
 
@@ -44,7 +44,7 @@ def _joblet_format(instance, job, width=74, jobid_type='agg'):
44
44
  mem = round(joblet.mem / 1024)
45
45
  joblet_repr += _format_to(f'{mem}G', 4, 'right')
46
46
  joblet_repr += ' '
47
- if instance.view_mode in ('gpu', 'ram'):
47
+ if instance.view_mode in ('gpu', 'ram', 'info'):
48
48
  joblet_repr += _format_to((f'{joblet.n_gpus}gp' if joblet.n_gpus > 0 else " - "), 3, 'left')
49
49
  elif instance.view_mode == 'cpu':
50
50
  joblet_repr += _format_to((f'{joblet.cpus}cp' if joblet.cpus > 0 else " - "), 3, 'left')
@@ -101,6 +101,20 @@ cpu_paused = '▚'
101
101
  cpu_down = '⨯'
102
102
 
103
103
  GLOBAL_NODES = None
104
+ BASE_PANEL_WIDTH = 33
105
+
106
+
107
+ def _center_panel_content(text, panel_width, base_width=BASE_PANEL_WIDTH):
108
+ if panel_width is None or panel_width <= base_width:
109
+ return text
110
+
111
+ pad = max(0, (panel_width - base_width) // 2)
112
+ if pad == 0:
113
+ return text
114
+
115
+ lines = text.split('\n')
116
+ lines = [(' ' * pad + line) if line else line for line in lines]
117
+ return '\n'.join(lines)
104
118
 
105
119
 
106
120
  def get_ram_block(megs):
@@ -138,15 +152,15 @@ def get_node_status(node, jobs):
138
152
  return None # white?
139
153
 
140
154
 
141
- def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu'):
155
+ def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu', panel_width=None):
142
156
  if mode == 'gpu':
143
- return view_viz_gpu(infrastructure, jobs, work, stylefn, current_user)
157
+ return _center_panel_content(view_viz_gpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
144
158
  elif mode == 'ram':
145
- return view_viz_ram(infrastructure, jobs, work, stylefn, current_user)
159
+ return _center_panel_content(view_viz_ram(infrastructure, jobs, work, stylefn, current_user), panel_width)
146
160
  elif mode == 'cpu':
147
- return view_viz_cpu(infrastructure, jobs, work, stylefn, current_user)
161
+ return _center_panel_content(view_viz_cpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
148
162
  elif mode == 'info':
149
- return view_viz_info(infrastructure, jobs, work, stylefn, current_user)
163
+ return view_viz_info(infrastructure, jobs, work, stylefn, current_user, panel_width)
150
164
 
151
165
 
152
166
  def view_viz_ram(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
@@ -519,7 +533,7 @@ def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user
519
533
  return RetScope.return_string
520
534
 
521
535
 
522
- def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
536
+ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, panel_width=None):
523
537
  # this is for hot reload
524
538
  if not work:
525
539
  return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
@@ -555,6 +569,7 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
555
569
  # max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
556
570
  feat_str = stylefn("GREEN", "FEATURES")
557
571
  RetScope.return_string += f'{"":{gpu_name_chars}} GPU {feat_str}\n'
572
+
558
573
  for i, n in GLOBAL_NODES.iterrows():
559
574
  gpu_model = n.m_gpu # if n.m_gpu is not None else 'N/A'
560
575
  # gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes