nodeocc 1.0.25__tar.gz → 1.0.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {nodeocc-1.0.25 → nodeocc-1.0.27}/PKG-INFO +5 -1
  2. {nodeocc-1.0.25 → nodeocc-1.0.27}/README.md +4 -0
  3. {nodeocc-1.0.25 → nodeocc-1.0.27}/pyproject.toml +1 -1
  4. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/controller/controller.py +36 -50
  5. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/PKG-INFO +5 -1
  6. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/SOURCES.txt +1 -0
  7. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/readers/slurmreader.py +21 -7
  8. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/__init__.py +7 -1
  9. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/curses_multiwindow.py +118 -9
  10. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/slurm_list.py +1 -1
  11. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/slurm_viz.py +21 -6
  12. nodeocc-1.0.27/src/view/transport.py +46 -0
  13. {nodeocc-1.0.25 → nodeocc-1.0.27}/setup.cfg +0 -0
  14. {nodeocc-1.0.25 → nodeocc-1.0.27}/setup.py +0 -0
  15. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/__init__.py +0 -0
  16. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/model/infrastructure.py +0 -0
  17. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/model/job.py +0 -0
  18. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/dependency_links.txt +0 -0
  19. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/entry_points.txt +0 -0
  20. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/requires.txt +0 -0
  21. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/nodeocc.egg-info/top_level.txt +0 -0
  22. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/readers/compute_wait_time.py +0 -0
  23. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/styles.py +0 -0
  24. {nodeocc-1.0.25 → nodeocc-1.0.27}/src/view/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nodeocc
3
- Version: 1.0.25
3
+ Version: 1.0.27
4
4
  Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
5
5
  Author: Matteo Boschini
6
6
  Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -31,4 +31,8 @@ Contents:
31
31
 
32
32
  Run it with: `python controller/controller.py`
33
33
 
34
+ Press `D` in the interactive view to open or close the debug screen. It shows
35
+ the local role, master PID and port, snapshot status, and local package
36
+ versions. Use the up and down keys to scroll.
37
+
34
38
  Requirements: pandas, pycurses.
@@ -13,4 +13,8 @@ Contents:
13
13
 
14
14
  Run it with: `python controller/controller.py`
15
15
 
16
+ Press `D` in the interactive view to open or close the debug screen. It shows
17
+ the local role, master PID and port, snapshot status, and local package
18
+ versions. Use the up and down keys to scroll.
19
+
16
20
  Requirements: pandas, pycurses.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "nodeocc"
7
- version = "1.0.25"
7
+ version = "1.0.27"
8
8
  authors = [{ name = "Matteo Boschini" }]
9
9
  maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
10
10
  description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"
@@ -24,43 +24,38 @@ import json
24
24
  import argparse
25
25
  import traceback
26
26
  import socket
27
+ import re
28
+ from urllib.request import urlopen
27
29
  from model.infrastructure import Infrastructure
28
30
  from model.job import Job
29
31
  import asyncio
30
32
  import importlib.metadata
33
+ from view.transport import send as send_datagram, receive as receive_datagram
31
34
 
32
35
  program_name = 'nodeocc'
33
36
  version_number = importlib.metadata.version(program_name)
34
- updated = False
35
37
 
36
- BBLUE = '\033[1;34m'
37
- END = '\033[0m'
38
- try:
39
- url = "https://pypi.python.org/pypi/nodeocc/json"
40
- import requests
41
- newest_version = json.loads(requests.get(url).text)['info']['version']
42
- print(BBLUE, end='')
43
- if int(newest_version.split('.')[0]) > int(version_number.split('.')[0]):
44
- print(f"New major version available: {newest_version} (current: {version_number})")
45
- print(f"Please update with 'pip install {program_name} --upgrade'")
46
- time.sleep(7)
47
- elif int(newest_version.split('.')[1]) > int(version_number.split('.')[1]):
48
- print(f"New minor version available: {newest_version} (current: {version_number})")
49
- print(f"Please update with 'pip install {program_name} --upgrade'")
50
- time.sleep(7)
51
- elif int(newest_version.split('.')[2]) > int(version_number.split('.')[2]):
52
- print(f"New patch version available: {newest_version} (current: {version_number})")
53
- print(f"Please update with 'pip install {program_name} --upgrade'")
54
- time.sleep(7)
55
- else:
56
- print(f"Up to date: {version_number}")
57
- updated = True
58
- time.sleep(3)
59
- print(END, end='')
60
- except Exception as e:
61
- print(f"{BBLUE}Could not get newest version from pip, check if {program_name} is up to date{END}")
62
- newest_version = '?.?.?'
63
- time.sleep(5)
38
+
39
+ def _version_tuple(v):
40
+ return tuple(int(x) for x in re.findall(r'\d+', str(v)))
41
+
42
+
43
+ def check_latest_version(cur_version, timeout_sec=1.2):
44
+ """
45
+ Fast, non-blocking-ish version check:
46
+ - short timeout
47
+ - no startup sleeps/pauses
48
+ - graceful fallback to current version on errors
49
+ """
50
+ try:
51
+ url = f"https://pypi.python.org/pypi/{program_name}/json"
52
+ with urlopen(url, timeout=timeout_sec) as response:
53
+ payload = json.loads(response.read().decode('utf-8'))
54
+ newest = payload['info']['version']
55
+ is_updated = _version_tuple(cur_version) >= _version_tuple(newest)
56
+ return newest, is_updated
57
+ except Exception:
58
+ return cur_version, True
64
59
 
65
60
  BEGIN_DELIM = "!{$"
66
61
  END_DELIM_ENCODED = "!}$".encode('utf-8')
@@ -81,6 +76,12 @@ args = parser.parse_args()
81
76
  # check basepath is readable
82
77
  assert os.access(args.basepath, os.R_OK), f"Base path {args.basepath} not readable"
83
78
 
79
+ # Keep startup snappy for daemon/diagnostics; check updates only in interactive mode.
80
+ if args.master_only or args.diag:
81
+ newest_version, updated = version_number, True
82
+ else:
83
+ newest_version, updated = check_latest_version(version_number)
84
+
84
85
  # export args
85
86
  orig_instance = Singleton.getInstance(args)
86
87
 
@@ -121,7 +122,7 @@ def send_msg(instance: Singleton, msg, n_extra_msgs=0, is_extra=False):
121
122
  msg = (str(msg_len) + begin_delim).encode('utf-8') + msg + END_DELIM_ENCODED
122
123
  instance.timeme(f"- msg encoded with len {msg_len}")
123
124
 
124
- instance.sock.sendto(msg, ('<broadcast>', instance.port))
125
+ send_datagram(instance.sock, instance.port, msg)
125
126
 
126
127
 
127
128
  def update_data_master(instance: Singleton):
@@ -139,24 +140,11 @@ def update_data_master(instance: Singleton):
139
140
  running_jobs = [j.to_nested_dict() for j in jobs if j.state == 'CG' or j.state == 'R']
140
141
  queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
141
142
  queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
142
- maxlen = min(JOB_LIMIT_PER_MSG, len(running_jobs) + len(queue_jobs))
143
-
144
- queue_jobs, extra_queue_jobs = queue_jobs[:maxlen - len(running_jobs)], queue_jobs[maxlen - len(running_jobs):]
145
143
  cur_timestamp = time.time()
146
144
  msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs, 'ts': str(cur_timestamp)})
147
-
148
- N_EXTRA_MSGS = (len(extra_queue_jobs) // JOB_LIMIT_PER_MSG) + 1 if len(extra_queue_jobs) > 0 else 0
149
- send_msg(instance, msg, n_extra_msgs=N_EXTRA_MSGS)
145
+ send_msg(instance, msg)
150
146
  instance.timeme("- main broadcast")
151
147
 
152
- if extra_queue_jobs:
153
- # get batches of JOB_LIMIT_PER_MSG jobs
154
- for job_idx in range(N_EXTRA_MSGS):
155
- i = job_idx * JOB_LIMIT_PER_MSG
156
- msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': extra_queue_jobs[i:i + JOB_LIMIT_PER_MSG], 'ts': str(cur_timestamp)})
157
- send_msg(instance, msg, is_extra=True, n_extra_msgs=N_EXTRA_MSGS - job_idx - 1)
158
- instance.timeme(f"- extra broadcast #{job_idx+1}")
159
-
160
148
  return inf, jobs
161
149
 
162
150
 
@@ -178,7 +166,7 @@ async def get_data_slave(instance):
178
166
  try:
179
167
  # listen for data from master
180
168
  # instance.sock.settimeout(6.5)
181
- data, _ = instance.sock.recvfrom(MAX_BUF)
169
+ data = receive_datagram(instance.sock, MAX_BUF)
182
170
  decoded_data = data.decode('utf-8', errors='replace')
183
171
  if BEGIN_DELIM not in decoded_data:
184
172
  if EXTRA_MSG_BEGIN_DELIM in decoded_data:
@@ -198,7 +186,7 @@ async def get_data_slave(instance):
198
186
  total_msg_n = n_msgs_remaining
199
187
  msg_store = {i: [] for i in range(total_msg_n)}
200
188
  while n_msgs_remaining > 0:
201
- data, _ = instance.sock.recvfrom(MAX_BUF)
189
+ data = receive_datagram(instance.sock, MAX_BUF)
202
190
  decoded_extra = data.decode('utf-8', errors='replace')
203
191
  if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
204
192
  instance.log("Received non-extra packet while waiting for extra packets, skipping")
@@ -239,8 +227,6 @@ async def get_data_slave(instance):
239
227
  if n_timeout > TIMEOUT_MAX:
240
228
  try_open_socket_as_slave(instance, force=True)
241
229
 
242
- time.sleep(random.randint(100, 10000) / 1000)
243
-
244
230
  except Exception as e:
245
231
  instance.err(f"Exception while receiving data from master: {e}")
246
232
  instance.err(traceback.format_exc())
@@ -272,7 +258,6 @@ async def get_all():
272
258
  if orig_instance.is_master:
273
259
  inf, jobs = update_data_master(orig_instance)
274
260
 
275
- time.sleep(5)
276
261
  else:
277
262
  # loop = asyncio.get_event_loop()
278
263
  orig_instance.timeme(f"- listening for data")
@@ -345,6 +330,7 @@ def _main():
345
330
  orig_instance.log(f"Starting master daemon")
346
331
 
347
332
  while True:
333
+ cycle_start = time.monotonic()
348
334
  orig_instance.timeme(f"Updating...")
349
335
  try:
350
336
  update_data_master(orig_instance)
@@ -352,7 +338,7 @@ def _main():
352
338
  orig_instance.err(f"Exception: {e}")
353
339
  orig_instance.err(traceback.format_exc())
354
340
 
355
- time.sleep(5)
341
+ time.sleep(max(0, 5 - (time.monotonic() - cycle_start)))
356
342
 
357
343
  else:
358
344
  # configure singleton
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nodeocc
3
- Version: 1.0.25
3
+ Version: 1.0.27
4
4
  Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
5
5
  Author: Matteo Boschini
6
6
  Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -31,4 +31,8 @@ Contents:
31
31
 
32
32
  Run it with: `python controller/controller.py`
33
33
 
34
+ Press `D` in the interactive view to open or close the debug screen. It shows
35
+ the local role, master PID and port, snapshot status, and local package
36
+ versions. Use the up and down keys to scroll.
37
+
34
38
  Requirements: pandas, pycurses.
@@ -18,4 +18,5 @@ src/view/curses_multiwindow.py
18
18
  src/view/slurm_list.py
19
19
  src/view/slurm_viz.py
20
20
  src/view/styles.py
21
+ src/view/transport.py
21
22
  src/view/utils.py
@@ -1,5 +1,6 @@
1
1
  import os
2
2
  import re
3
+ from collections import defaultdict
3
4
  from io import StringIO
4
5
 
5
6
  import pandas as pd
@@ -57,9 +58,16 @@ def _parse_gpu_count(gres):
57
58
  token = token.strip()
58
59
  if not token.lower().startswith('gpu'):
59
60
  continue
60
- last_field = token.split(':')[-1].split('(')[0]
61
- match = re.search(r'\d+', last_field)
62
- n_gpu += int(match.group(0)) if match else 1
61
+ token_core = token.split('(')[0]
62
+ parts = token_core.split(':')
63
+ count = None
64
+
65
+ if len(parts) >= 2 and re.fullmatch(r'\d+', parts[-1]):
66
+ count = int(parts[-1])
67
+ elif len(parts) >= 3 and re.fullmatch(r'\d+', parts[-2]):
68
+ count = int(parts[-2])
69
+
70
+ n_gpu += count if count is not None else 1
63
71
  return n_gpu
64
72
 
65
73
 
@@ -75,10 +83,13 @@ def _parse_gpu_models(gres):
75
83
  token = token.strip()
76
84
  if not token.lower().startswith('gpu'):
77
85
  continue
78
- parts = token.split(':')
79
- if len(parts) >= 3:
86
+ token_core = token.split('(')[0]
87
+ parts = token_core.split(':')
88
+ if len(parts) >= 3 and not re.fullmatch(r'\d+', parts[1]):
89
+ model = parts[1]
90
+ elif len(parts) == 2 and not re.fullmatch(r'\d+', parts[1]):
80
91
  model = parts[1]
81
- elif len(parts) == 2:
92
+ elif len(parts) == 2 and re.fullmatch(r'\d+', parts[1]):
82
93
  model = 'gpu'
83
94
  else:
84
95
  model = 'n/a'
@@ -367,8 +378,11 @@ def read_jobs():
367
378
  line['NICE']
368
379
  ), axis=1).tolist()
369
380
 
381
+ joblets_by_job = defaultdict(list)
382
+ for joblet in joblets:
383
+ joblets_by_job[joblet.jobid].append(joblet)
370
384
  for j in jobs:
371
- j.joblets = [x for x in joblets if x.jobid == j.jobid]
385
+ j.joblets = joblets_by_job[j.jobid]
372
386
  return jobs, joblets
373
387
 
374
388
 
@@ -13,5 +13,11 @@ def update_views(stdscr, instance, filter):
13
13
  instance, 'job_id_type') else 'agg')
14
14
  instance.timeme(f"- print job list")
15
15
 
16
- instance.nocc = slurm_viz.view_viz(instance.inf, instance.jobs, stylefn=styles.crsstyler, mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu')
16
+ instance.nocc = slurm_viz.view_viz(
17
+ instance.inf,
18
+ instance.jobs,
19
+ stylefn=styles.crsstyler,
20
+ mode=instance.view_mode if hasattr(instance, 'view_mode') else 'gpu',
21
+ panel_width=instance.right_width if hasattr(instance, 'right_width') else None,
22
+ )
17
23
  instance.timeme(f"- view nodeocc")
@@ -14,6 +14,9 @@ from logging.handlers import RotatingFileHandler
14
14
  from pathlib import Path
15
15
  from view import update_views
16
16
  import stat
17
+ import sys
18
+ from datetime import datetime
19
+ from importlib import metadata
17
20
 
18
21
  a_filter_values = [None, 'me', 'prod', 'stud', 'cvcs']
19
22
 
@@ -33,8 +36,14 @@ BUTTON_ACTIONS = {
33
36
  'show_prio': [ord('p')],
34
37
  'refresh': [ord('y')],
35
38
  'info': [ord('i')],
39
+ 'debug_view': [ord('D')],
36
40
  }
37
41
 
42
+ RIGHT_WIDTH_GPU = 33
43
+ RIGHT_WIDTH_INFO = 39
44
+ # Keep a single stable width across modes; use the widest one so info fields are not cut.
45
+ RIGHT_WIDTH_UNIFIED = max(RIGHT_WIDTH_GPU, RIGHT_WIDTH_INFO)
46
+
38
47
 
39
48
  def is_file_writable_byall(file_path):
40
49
  return os.stat(file_path).st_mode & stat.S_IWOTH
@@ -48,7 +57,10 @@ def is_nodeocc_process(pid):
48
57
  try:
49
58
  cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
50
59
  return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
51
- except Exception:
60
+ except psutil.AccessDenied:
61
+ # Other users' command lines may be hidden on shared login nodes.
62
+ return psutil.pid_exists(int(pid))
63
+ except (psutil.Error, ValueError):
52
64
  return False
53
65
 
54
66
 
@@ -67,7 +79,10 @@ def process_has_udp_port(pid, port):
67
79
  for conn in proc.connections(kind='udp'):
68
80
  if conn.laddr and conn.laddr.port == int(port):
69
81
  return True
70
- except Exception:
82
+ except psutil.AccessDenied:
83
+ # Socket ownership is commonly visible only to the process owner.
84
+ return psutil.pid_exists(int(pid))
85
+ except (psutil.Error, ValueError):
71
86
  return False
72
87
  return False
73
88
 
@@ -169,6 +184,11 @@ class Singleton:
169
184
  self.rens = ""
170
185
 
171
186
  self.fetch_fn = None
187
+ self.debug_view = False
188
+ self.debug_offset = 0
189
+ self.last_success = None
190
+ self.last_fetch_duration = None
191
+ self.last_error = None
172
192
  self.view_mode = 'gpu'
173
193
  self.job_id_type = 'agg'
174
194
 
@@ -319,6 +339,7 @@ class Singleton:
319
339
  self._ctime = _ntime
320
340
 
321
341
  def err(self, msg):
342
+ self.last_error = str(msg)
322
343
  if self.args.debug or self.args.master_only:
323
344
  logging.error(msg)
324
345
 
@@ -332,11 +353,14 @@ class Singleton:
332
353
  if self.fetch_fn is not None:
333
354
  inf, jobs, avg_wait_time = await self.fetch_fn() # a_filter_values[self.a_filter])
334
355
  master_dead = inf is None
356
+ if not master_dead:
357
+ self.last_success = time.time()
335
358
  self.inf = inf if inf is not None else self.inf
336
359
  self.jobs = jobs if jobs is not None else self.jobs
337
360
  self.avg_wait_time = avg_wait_time if avg_wait_time is not None else self.avg_wait_time
338
361
 
339
362
  _delta_t = time.time() - _ctime
363
+ self.last_fetch_duration = _delta_t
340
364
  self.log(f"Fetch took {_delta_t:.2f} seconds")
341
365
  return master_dead
342
366
 
@@ -454,6 +478,18 @@ def handle_keys(stdscr, instance):
454
478
 
455
479
  instance.mouse_state = {}
456
480
 
481
+ if k in BUTTON_ACTIONS['debug_view']:
482
+ instance.debug_view = not instance.debug_view
483
+ instance.debug_offset = 0
484
+ stdscr.clear()
485
+ return
486
+ if instance.debug_view:
487
+ if k in BUTTON_ACTIONS['down_1']:
488
+ instance.debug_offset += 1
489
+ elif k in BUTTON_ACTIONS['up_1']:
490
+ instance.debug_offset = max(0, instance.debug_offset - 1)
491
+ return
492
+
457
493
  # process input
458
494
  # RIGHT
459
495
  if k in BUTTON_ACTIONS['right_1']:
@@ -476,7 +512,7 @@ def handle_keys(stdscr, instance):
476
512
  instance.sort_by_prio = not instance.sort_by_prio
477
513
  elif k in BUTTON_ACTIONS['res_view_mode']:
478
514
  instance.view_mode = {"gpu": "ram", "ram": "cpu", "cpu": "gpu", "info": "gpu"}[instance.view_mode]
479
- instance.right_width = 33 if instance.view_mode != 'info' else 39
515
+ instance.right_width = RIGHT_WIDTH_UNIFIED
480
516
  stdscr.clear()
481
517
  elif k in BUTTON_ACTIONS['job_id_type']:
482
518
  instance.job_id_type = "true" if instance.job_id_type == "agg" else "agg"
@@ -489,7 +525,7 @@ def handle_keys(stdscr, instance):
489
525
  instance.show_prio = not instance.show_prio
490
526
  elif k in BUTTON_ACTIONS['info']:
491
527
  instance.view_mode = 'info' if instance.view_mode != 'info' else 'gpu'
492
- instance.right_width = 33 if instance.view_mode != 'info' else 39
528
+ instance.right_width = RIGHT_WIDTH_UNIFIED
493
529
  stdscr.clear()
494
530
  elif k in BUTTON_ACTIONS['up_tab']:
495
531
  # get screen size
@@ -502,7 +538,70 @@ def handle_keys(stdscr, instance):
502
538
  instance.voff -= ((height - 4) // 2) + 1
503
539
 
504
540
 
541
+ def debug_lines(instance):
542
+ """Collect local diagnostics without querying Slurm or the network."""
543
+ if not hasattr(instance, '_package_versions'):
544
+ instance._package_versions = {}
545
+ for name in ('nodeocc', 'pandas', 'numpy', 'psutil'):
546
+ try:
547
+ instance._package_versions[name] = metadata.version(name)
548
+ except metadata.PackageNotFoundError:
549
+ instance._package_versions[name] = 'not installed'
550
+ port = getattr(instance, 'port', None)
551
+ master_pid = str(os.getpid()) if instance.is_master else 'unknown'
552
+ try:
553
+ for path in instance.get_port_files():
554
+ if port_from_portfile(path) == port:
555
+ master_pid = Path(path).read_text().strip() or 'unknown'
556
+ break
557
+ except OSError:
558
+ pass
559
+ try:
560
+ master_owner = psutil.Process(int(master_pid)).username()
561
+ except (ValueError, psutil.Error, OSError, KeyError):
562
+ master_owner = 'unknown'
563
+ last_success = (datetime.fromtimestamp(instance.last_success).strftime('%Y-%m-%d %H:%M:%S')
564
+ if instance.last_success else 'never')
565
+ duration = (f'{instance.last_fetch_duration:.2f} s'
566
+ if instance.last_fetch_duration is not None else 'not available')
567
+ lines = [
568
+ 'Connection',
569
+ f' Host: {socket.getfqdn()}',
570
+ f' Role: {"master" if instance.is_master else "client"}',
571
+ f' Master PID: {master_pid}',
572
+ f' Master owner: {master_owner}',
573
+ f' Master UDP port: {port if port is not None else "unknown"}',
574
+ f' Base path: {instance.basepath}',
575
+ '', 'Snapshot',
576
+ f' Last successful update: {last_success}',
577
+ f' Last {"update" if instance.is_master else "receive wait"}: {duration}',
578
+ f' Jobs: {len(instance.jobs) if instance.jobs is not None else "unknown"}',
579
+ f' Nodes: {len(instance.inf.nodes) if instance.inf is not None else "unknown"}',
580
+ f' Last error: {instance.last_error or "none recorded"}',
581
+ '', 'Environment',
582
+ f' Python: {sys.version.split()[0]}',
583
+ f' Executable: {sys.executable}',
584
+ f' Process PID: {os.getpid()}',
585
+ ' Package versions (local environment):',
586
+ ]
587
+ lines.extend(f' {name}: {version}' for name, version in instance._package_versions.items())
588
+ return lines
589
+
590
+
505
591
  def update_screen(stdscr, instance):
592
+ if instance.debug_view:
593
+ lines, columns = stdscr.getmaxyx()
594
+ content = debug_lines(instance)
595
+ instance.debug_offset = min(instance.debug_offset, max(0, len(content) - max(1, lines - 2)))
596
+ stdscr.erase()
597
+ try:
598
+ stdscr.addnstr(0, 0, 'DEBUG [D: close] [up/down: scroll] [Q: quit]', max(0, columns - 1))
599
+ for row, line in enumerate(content[instance.debug_offset:instance.debug_offset + max(0, lines - 2)], 1):
600
+ stdscr.addnstr(row, 0, line, max(0, columns - 1))
601
+ except curses.error:
602
+ pass
603
+ stdscr.refresh()
604
+ return
506
605
  update_views(stdscr, instance, a_filter_values[instance.a_filter])
507
606
  xoffset = 0
508
607
  instance.xoffset = xoffset
@@ -513,7 +612,7 @@ def update_screen(stdscr, instance):
513
612
  if instance.k in BUTTON_ACTIONS['refresh']:
514
613
  stdscr.clear()
515
614
 
516
- totsize = 106
615
+ totsize = 106 + max(0, RIGHT_WIDTH_UNIFIED - RIGHT_WIDTH_GPU)
517
616
  if instance.show_account:
518
617
  totsize += 10
519
618
  if instance.show_prio:
@@ -579,7 +678,9 @@ def update_screen(stdscr, instance):
579
678
  stdscr.addstr(0, left_width + 5, 'INFO', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'info' else 0))
580
679
  stdscr.addstr(0, left_width + 9, ']', curses.color_pair(2))
581
680
  instance.add_button(0, left_width + 2, '[I:INFO]', BUTTON_ACTIONS['info'][0])
582
- stdscr.addstr(0, left_width + 10, '─' * (columns - 15 - left_width - 3), curses.color_pair(2))
681
+ stdscr.addstr(0, left_width + 11, '[D:DEBUG]', curses.color_pair(2))
682
+ instance.add_button(0, left_width + 11, '[D:DEBUG]', BUTTON_ACTIONS['debug_view'][0])
683
+ stdscr.addstr(0, left_width + 20, '─' * max(0, columns - 15 - left_width - 20), curses.color_pair(2))
583
684
  stdscr.addstr(0, columns - 15, '[G:', curses.color_pair(2))
584
685
  stdscr.addstr(0, columns - 12, 'GPU', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'gpu' else 0))
585
686
  stdscr.addstr(0, columns - 12 + 3, 'RAM', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'ram' else 0))
@@ -606,8 +707,14 @@ def update_screen(stdscr, instance):
606
707
  # get slurm user partition
607
708
  # stdscr.addstr(lines - 1, xoffset + 62 + 2, f'(Avg time {instance.avg_wait_time})', curses.color_pair(2))
608
709
 
609
- signature = instance.signature if instance.updated else f'{instance.version} -> {instance.newest_version}'
610
- stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, curses.color_pair(2) if instance.updated else curses.color_pair(3))
710
+ if instance.updated:
711
+ signature = instance.signature
712
+ sig_style = curses.color_pair(2)
713
+ else:
714
+ signature = f'{instance.signature} (new {instance.newest_version})'
715
+ sig_style = curses.color_pair(4)
716
+
717
+ stdscr.addstr(lines - 1, columns - 2 - len(signature), signature, sig_style)
611
718
 
612
719
  stdscr.refresh()
613
720
  curses.doupdate()
@@ -645,6 +752,8 @@ async def update_screen_info(stdscr, instance: Singleton):
645
752
  try_open_socket_as_slave(instance)
646
753
 
647
754
  update_screen(stdscr, instance)
755
+ if instance.is_master:
756
+ await asyncio.sleep(max(0, 5 - (instance.last_fetch_duration or 0)))
648
757
 
649
758
 
650
759
  async def wait_first(futures, instance):
@@ -712,7 +821,7 @@ async def main(stdscr):
712
821
  # status
713
822
 
714
823
  instance.a_filter = 0
715
- instance.right_width = 33
824
+ instance.right_width = RIGHT_WIDTH_UNIFIED
716
825
 
717
826
  stdscr.clear()
718
827
 
@@ -44,7 +44,7 @@ def _joblet_format(instance, job, width=74, jobid_type='agg'):
44
44
  mem = round(joblet.mem / 1024)
45
45
  joblet_repr += _format_to(f'{mem}G', 4, 'right')
46
46
  joblet_repr += ' '
47
- if instance.view_mode in ('gpu', 'ram'):
47
+ if instance.view_mode in ('gpu', 'ram', 'info'):
48
48
  joblet_repr += _format_to((f'{joblet.n_gpus}gp' if joblet.n_gpus > 0 else " - "), 3, 'left')
49
49
  elif instance.view_mode == 'cpu':
50
50
  joblet_repr += _format_to((f'{joblet.cpus}cp' if joblet.cpus > 0 else " - "), 3, 'left')
@@ -101,6 +101,20 @@ cpu_paused = '▚'
101
101
  cpu_down = '⨯'
102
102
 
103
103
  GLOBAL_NODES = None
104
+ BASE_PANEL_WIDTH = 33
105
+
106
+
107
+ def _center_panel_content(text, panel_width, base_width=BASE_PANEL_WIDTH):
108
+ if panel_width is None or panel_width <= base_width:
109
+ return text
110
+
111
+ pad = max(0, (panel_width - base_width) // 2)
112
+ if pad == 0:
113
+ return text
114
+
115
+ lines = text.split('\n')
116
+ lines = [(' ' * pad + line) if line else line for line in lines]
117
+ return '\n'.join(lines)
104
118
 
105
119
 
106
120
  def get_ram_block(megs):
@@ -138,15 +152,15 @@ def get_node_status(node, jobs):
138
152
  return None # white?
139
153
 
140
154
 
141
- def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu'):
155
+ def view_viz(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, mode='gpu', panel_width=None):
142
156
  if mode == 'gpu':
143
- return view_viz_gpu(infrastructure, jobs, work, stylefn, current_user)
157
+ return _center_panel_content(view_viz_gpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
144
158
  elif mode == 'ram':
145
- return view_viz_ram(infrastructure, jobs, work, stylefn, current_user)
159
+ return _center_panel_content(view_viz_ram(infrastructure, jobs, work, stylefn, current_user), panel_width)
146
160
  elif mode == 'cpu':
147
- return view_viz_cpu(infrastructure, jobs, work, stylefn, current_user)
161
+ return _center_panel_content(view_viz_cpu(infrastructure, jobs, work, stylefn, current_user), panel_width)
148
162
  elif mode == 'info':
149
- return view_viz_info(infrastructure, jobs, work, stylefn, current_user)
163
+ return view_viz_info(infrastructure, jobs, work, stylefn, current_user, panel_width)
150
164
 
151
165
 
152
166
  def view_viz_ram(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
@@ -519,7 +533,7 @@ def view_viz_cpu(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user
519
533
  return RetScope.return_string
520
534
 
521
535
 
522
- def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None):
536
+ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_user=None, panel_width=None):
523
537
  # this is for hot reload
524
538
  if not work:
525
539
  return "UPDATE IN PROGRESS - PLZ W8 M8 B8"
@@ -555,6 +569,7 @@ def view_viz_info(infrastructure, jobs, work=True, stylefn=cmdstyle, current_use
555
569
  # max_ram_len = max([len(str(n.mem // 1024)) for n in nodes])
556
570
  feat_str = stylefn("GREEN", "FEATURES")
557
571
  RetScope.return_string += f'{"":{gpu_name_chars}} GPU {feat_str}\n'
572
+
558
573
  for i, n in GLOBAL_NODES.iterrows():
559
574
  gpu_model = n.m_gpu # if n.m_gpu is not None else 'N/A'
560
575
  # gpu_mem = n.gpu_mem if n.gpu_mem is not None else 'N/A'
@@ -0,0 +1,46 @@
1
+ """Fragment large broadcast messages to fit in UDP datagrams."""
2
+ import uuid
3
+ import zlib
4
+
5
+ PREFIX = b'nodeocc-fragment:'
6
+ CHUNK_SIZE = 60000
7
+ COMPRESSED = b'nodeocc-zlib:'
8
+
9
+
10
+ def send(sock, port, data):
11
+ compressed = COMPRESSED + zlib.compress(data, level=1)
12
+ if len(compressed) < len(data):
13
+ data = compressed
14
+ if len(data) <= CHUNK_SIZE:
15
+ sock.sendto(data, ('<broadcast>', port))
16
+ return
17
+ message_id = uuid.uuid4().hex.encode('ascii')
18
+ count = (len(data) + CHUNK_SIZE - 1) // CHUNK_SIZE
19
+ for index in range(count):
20
+ header = PREFIX + message_id + b':' + str(index).encode() + b':' + str(count).encode() + b':'
21
+ sock.sendto(header + data[index * CHUNK_SIZE:(index + 1) * CHUNK_SIZE], ('<broadcast>', port))
22
+
23
+
24
+ def receive(sock, max_buf):
25
+ fragments = {}
26
+ while True:
27
+ data, _ = sock.recvfrom(max_buf)
28
+ if not data.startswith(PREFIX):
29
+ return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data
30
+ try:
31
+ message_id, index, count, chunk = data[len(PREFIX):].split(b':', 3)
32
+ index, count = int(index), int(count)
33
+ if count < 2 or count > 10000 or not 0 <= index < count:
34
+ continue
35
+ except ValueError:
36
+ continue
37
+ # Retain only one message, so lost fragments cannot accumulate forever.
38
+ if message_id not in fragments:
39
+ fragments = {message_id: (count, {})}
40
+ expected, parts = fragments[message_id]
41
+ if count != expected:
42
+ continue
43
+ parts[index] = chunk
44
+ if len(parts) == count:
45
+ data = b''.join(parts[i] for i in range(count))
46
+ return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes