nodeocc 1.0.26__tar.gz → 1.0.27__tar.gz

{nodeocc-1.0.26 → nodeocc-1.0.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nodeocc
-Version: 1.0.26
+Version: 1.0.27
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -31,4 +31,8 @@ Contents:
 
 Run it with: `python controller/controller.py`
 
+Press `D` in the interactive view to open or close the debug screen. It shows
+the local role, master PID and port, snapshot status, and local package
+versions. Use the up and down keys to scroll.
+
 Requirements: pandas, pycurses.

{nodeocc-1.0.26 → nodeocc-1.0.27}/README.md

@@ -13,4 +13,8 @@ Contents:
 
 Run it with: `python controller/controller.py`
 
+Press `D` in the interactive view to open or close the debug screen. It shows
+the local role, master PID and port, snapshot status, and local package
+versions. Use the up and down keys to scroll.
+
 Requirements: pandas, pycurses.

{nodeocc-1.0.26 → nodeocc-1.0.27}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.26"
+version = "1.0.27"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.26 → nodeocc-1.0.27}/src/controller/controller.py

@@ -30,6 +30,7 @@ from model.infrastructure import Infrastructure
 from model.job import Job
 import asyncio
 import importlib.metadata
+from view.transport import send as send_datagram, receive as receive_datagram
 
 program_name = 'nodeocc'
 version_number = importlib.metadata.version(program_name)
@@ -121,7 +122,7 @@ def send_msg(instance: Singleton, msg, n_extra_msgs=0, is_extra=False):
     msg = (str(msg_len) + begin_delim).encode('utf-8') + msg + END_DELIM_ENCODED
     instance.timeme(f"- msg encoded with len {msg_len}")
 
-    instance.sock.sendto(msg, ('<broadcast>', instance.port))
+    send_datagram(instance.sock, instance.port, msg)
 
 
 def update_data_master(instance: Singleton):
@@ -139,24 +140,11 @@ def update_data_master(instance: Singleton):
     running_jobs = [j.to_nested_dict() for j in jobs if j.state == 'CG' or j.state == 'R']
     queue_jobs = [j.to_nested_dict() for j in jobs if j.state != 'R' and j.state != 'CG']
     queue_jobs = sorted(queue_jobs, key=lambda x: x['priority'], reverse=True)
-    maxlen = min(JOB_LIMIT_PER_MSG, len(running_jobs) + len(queue_jobs))
-
-    queue_jobs, extra_queue_jobs = queue_jobs[:maxlen - len(running_jobs)], queue_jobs[maxlen - len(running_jobs):]
     cur_timestamp = time.time()
     msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': running_jobs + queue_jobs, 'ts': str(cur_timestamp)})
-
-    N_EXTRA_MSGS = (len(extra_queue_jobs) // JOB_LIMIT_PER_MSG) + 1 if len(extra_queue_jobs) > 0 else 0
-    send_msg(instance, msg, n_extra_msgs=N_EXTRA_MSGS)
+    send_msg(instance, msg)
     instance.timeme("- main broadcast")
 
-    if extra_queue_jobs:
-        # get batches of JOB_LIMIT_PER_MSG jobs
-        for job_idx in range(N_EXTRA_MSGS):
-            i = job_idx * JOB_LIMIT_PER_MSG
-            msg = json.dumps({'inf': inf.to_nested_dict(), 'jobs': extra_queue_jobs[i:i + JOB_LIMIT_PER_MSG], 'ts': str(cur_timestamp)})
-            send_msg(instance, msg, is_extra=True, n_extra_msgs=N_EXTRA_MSGS - job_idx - 1)
-            instance.timeme(f"- extra broadcast #{job_idx+1}")
-
     return inf, jobs
 
 
@@ -178,7 +166,7 @@ async def get_data_slave(instance):
     try:
         # listen for data from master
         # instance.sock.settimeout(6.5)
-        data, _ = instance.sock.recvfrom(MAX_BUF)
+        data = receive_datagram(instance.sock, MAX_BUF)
         decoded_data = data.decode('utf-8', errors='replace')
         if BEGIN_DELIM not in decoded_data:
             if EXTRA_MSG_BEGIN_DELIM in decoded_data:
@@ -198,7 +186,7 @@ async def get_data_slave(instance):
         total_msg_n = n_msgs_remaining
         msg_store = {i: [] for i in range(total_msg_n)}
         while n_msgs_remaining > 0:
-            data, _ = instance.sock.recvfrom(MAX_BUF)
+            data = receive_datagram(instance.sock, MAX_BUF)
             decoded_extra = data.decode('utf-8', errors='replace')
             if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
                 instance.log("Received non-extra packet while waiting for extra packets, skipping")
@@ -239,8 +227,6 @@ async def get_data_slave(instance):
         if n_timeout > TIMEOUT_MAX:
             try_open_socket_as_slave(instance, force=True)
 
-        time.sleep(random.randint(100, 10000) / 1000)
-
     except Exception as e:
         instance.err(f"Exception while receiving data from master: {e}")
         instance.err(traceback.format_exc())
@@ -272,7 +258,6 @@ async def get_all():
         if orig_instance.is_master:
             inf, jobs = update_data_master(orig_instance)
 
-            time.sleep(5)
         else:
             # loop = asyncio.get_event_loop()
             orig_instance.timeme(f"- listening for data")
@@ -345,6 +330,7 @@ def _main():
         orig_instance.log(f"Starting master daemon")
 
         while True:
+            cycle_start = time.monotonic()
             orig_instance.timeme(f"Updating...")
             try:
                 update_data_master(orig_instance)
@@ -352,7 +338,7 @@ def _main():
                 orig_instance.err(f"Exception: {e}")
                 orig_instance.err(traceback.format_exc())
 
-            time.sleep(5)
+            time.sleep(max(0, 5 - (time.monotonic() - cycle_start)))
 
     else:
         # configure singleton

{nodeocc-1.0.26 → nodeocc-1.0.27}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nodeocc
-Version: 1.0.26
+Version: 1.0.27
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello
@@ -31,4 +31,8 @@ Contents:
 
 Run it with: `python controller/controller.py`
 
+Press `D` in the interactive view to open or close the debug screen. It shows
+the local role, master PID and port, snapshot status, and local package
+versions. Use the up and down keys to scroll.
+
 Requirements: pandas, pycurses.

{nodeocc-1.0.26 → nodeocc-1.0.27}/src/nodeocc.egg-info/SOURCES.txt

@@ -18,4 +18,5 @@ src/view/curses_multiwindow.py
 src/view/slurm_list.py
 src/view/slurm_viz.py
 src/view/styles.py
+src/view/transport.py
 src/view/utils.py

{nodeocc-1.0.26 → nodeocc-1.0.27}/src/readers/slurmreader.py

@@ -1,5 +1,6 @@
 import os
 import re
+from collections import defaultdict
 from io import StringIO
 
 import pandas as pd
@@ -377,8 +378,11 @@ def read_jobs():
         line['NICE']
     ), axis=1).tolist()
 
+    joblets_by_job = defaultdict(list)
+    for joblet in joblets:
+        joblets_by_job[joblet.jobid].append(joblet)
     for j in jobs:
-        j.joblets = [x for x in joblets if x.jobid == j.jobid]
+        j.joblets = joblets_by_job[j.jobid]
     return jobs, joblets
 
 

{nodeocc-1.0.26 → nodeocc-1.0.27}/src/view/curses_multiwindow.py

@@ -14,6 +14,9 @@ from logging.handlers import RotatingFileHandler
 from pathlib import Path
 from view import update_views
 import stat
+import sys
+from datetime import datetime
+from importlib import metadata
 
 a_filter_values = [None, 'me', 'prod', 'stud', 'cvcs']
 
@@ -33,6 +36,7 @@ BUTTON_ACTIONS = {
     'show_prio': [ord('p')],
     'refresh': [ord('y')],
     'info': [ord('i')],
+    'debug_view': [ord('D')],
 }
 
 RIGHT_WIDTH_GPU = 33
@@ -53,7 +57,10 @@ def is_nodeocc_process(pid):
     try:
         cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
         return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
-    except Exception:
+    except psutil.AccessDenied:
+        # Other users' command lines may be hidden on shared login nodes.
+        return psutil.pid_exists(int(pid))
+    except (psutil.Error, ValueError):
         return False
 
 
@@ -72,7 +79,10 @@ def process_has_udp_port(pid, port):
         for conn in proc.connections(kind='udp'):
             if conn.laddr and conn.laddr.port == int(port):
                 return True
-    except Exception:
+    except psutil.AccessDenied:
+        # Socket ownership is commonly visible only to the process owner.
+        return psutil.pid_exists(int(pid))
+    except (psutil.Error, ValueError):
         return False
     return False
 
@@ -174,6 +184,11 @@ class Singleton:
         self.rens = ""
 
         self.fetch_fn = None
+        self.debug_view = False
+        self.debug_offset = 0
+        self.last_success = None
+        self.last_fetch_duration = None
+        self.last_error = None
         self.view_mode = 'gpu'
         self.job_id_type = 'agg'
 
@@ -324,6 +339,7 @@ class Singleton:
             self._ctime = _ntime
 
     def err(self, msg):
+        self.last_error = str(msg)
         if self.args.debug or self.args.master_only:
             logging.error(msg)
 
@@ -337,11 +353,14 @@ class Singleton:
         if self.fetch_fn is not None:
             inf, jobs, avg_wait_time = await self.fetch_fn()  # a_filter_values[self.a_filter])
             master_dead = inf is None
+            if not master_dead:
+                self.last_success = time.time()
             self.inf = inf if inf is not None else self.inf
             self.jobs = jobs if jobs is not None else self.jobs
             self.avg_wait_time = avg_wait_time if avg_wait_time is not None else self.avg_wait_time
 
         _delta_t = time.time() - _ctime
+        self.last_fetch_duration = _delta_t
         self.log(f"Fetch took {_delta_t:.2f} seconds")
         return master_dead
 
@@ -459,6 +478,18 @@ def handle_keys(stdscr, instance):
 
     instance.mouse_state = {}
 
+    if k in BUTTON_ACTIONS['debug_view']:
+        instance.debug_view = not instance.debug_view
+        instance.debug_offset = 0
+        stdscr.clear()
+        return
+    if instance.debug_view:
+        if k in BUTTON_ACTIONS['down_1']:
+            instance.debug_offset += 1
+        elif k in BUTTON_ACTIONS['up_1']:
+            instance.debug_offset = max(0, instance.debug_offset - 1)
+        return
+
     # process input
     # RIGHT
     if k in BUTTON_ACTIONS['right_1']:
@@ -507,7 +538,70 @@ def handle_keys(stdscr, instance):
         instance.voff -= ((height - 4) // 2) + 1
 
 
+def debug_lines(instance):
+    """Collect local diagnostics without querying Slurm or the network."""
+    if not hasattr(instance, '_package_versions'):
+        instance._package_versions = {}
+        for name in ('nodeocc', 'pandas', 'numpy', 'psutil'):
+            try:
+                instance._package_versions[name] = metadata.version(name)
+            except metadata.PackageNotFoundError:
+                instance._package_versions[name] = 'not installed'
+    port = getattr(instance, 'port', None)
+    master_pid = str(os.getpid()) if instance.is_master else 'unknown'
+    try:
+        for path in instance.get_port_files():
+            if port_from_portfile(path) == port:
+                master_pid = Path(path).read_text().strip() or 'unknown'
+                break
+    except OSError:
+        pass
+    try:
+        master_owner = psutil.Process(int(master_pid)).username()
+    except (ValueError, psutil.Error, OSError, KeyError):
+        master_owner = 'unknown'
+    last_success = (datetime.fromtimestamp(instance.last_success).strftime('%Y-%m-%d %H:%M:%S')
+                    if instance.last_success else 'never')
+    duration = (f'{instance.last_fetch_duration:.2f} s'
+                if instance.last_fetch_duration is not None else 'not available')
+    lines = [
+        'Connection',
+        f'  Host: {socket.getfqdn()}',
+        f'  Role: {"master" if instance.is_master else "client"}',
+        f'  Master PID: {master_pid}',
+        f'  Master owner: {master_owner}',
+        f'  Master UDP port: {port if port is not None else "unknown"}',
+        f'  Base path: {instance.basepath}',
+        '', 'Snapshot',
+        f'  Last successful update: {last_success}',
+        f'  Last {"update" if instance.is_master else "receive wait"}: {duration}',
+        f'  Jobs: {len(instance.jobs) if instance.jobs is not None else "unknown"}',
+        f'  Nodes: {len(instance.inf.nodes) if instance.inf is not None else "unknown"}',
+        f'  Last error: {instance.last_error or "none recorded"}',
+        '', 'Environment',
+        f'  Python: {sys.version.split()[0]}',
+        f'  Executable: {sys.executable}',
+        f'  Process PID: {os.getpid()}',
+        '  Package versions (local environment):',
+    ]
+    lines.extend(f'    {name}: {version}' for name, version in instance._package_versions.items())
+    return lines
+
+
 def update_screen(stdscr, instance):
+    if instance.debug_view:
+        lines, columns = stdscr.getmaxyx()
+        content = debug_lines(instance)
+        instance.debug_offset = min(instance.debug_offset, max(0, len(content) - max(1, lines - 2)))
+        stdscr.erase()
+        try:
+            stdscr.addnstr(0, 0, 'DEBUG  [D: close]  [up/down: scroll]  [Q: quit]', max(0, columns - 1))
+            for row, line in enumerate(content[instance.debug_offset:instance.debug_offset + max(0, lines - 2)], 1):
+                stdscr.addnstr(row, 0, line, max(0, columns - 1))
+        except curses.error:
+            pass
+        stdscr.refresh()
+        return
     update_views(stdscr, instance, a_filter_values[instance.a_filter])
     xoffset = 0
     instance.xoffset = xoffset
@@ -584,7 +678,9 @@ def update_screen(stdscr, instance):
     stdscr.addstr(0, left_width + 5, 'INFO', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'info' else 0))
     stdscr.addstr(0, left_width + 9, ']', curses.color_pair(2))
     instance.add_button(0, left_width + 2, '[I:INFO]', BUTTON_ACTIONS['info'][0])
-    stdscr.addstr(0, left_width + 10, '─' * (columns - 15 - left_width - 3), curses.color_pair(2))
+    stdscr.addstr(0, left_width + 11, '[D:DEBUG]', curses.color_pair(2))
+    instance.add_button(0, left_width + 11, '[D:DEBUG]', BUTTON_ACTIONS['debug_view'][0])
+    stdscr.addstr(0, left_width + 20, '─' * max(0, columns - 15 - left_width - 20), curses.color_pair(2))
     stdscr.addstr(0, columns - 15, '[G:', curses.color_pair(2))
     stdscr.addstr(0, columns - 12, 'GPU', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'gpu' else 0))
     stdscr.addstr(0, columns - 12 + 3, 'RAM', curses.color_pair(2) | (curses.A_REVERSE if instance.view_mode == 'ram' else 0))
@@ -656,6 +752,8 @@ async def update_screen_info(stdscr, instance: Singleton):
                 try_open_socket_as_slave(instance)
 
         update_screen(stdscr, instance)
+        if instance.is_master:
+            await asyncio.sleep(max(0, 5 - (instance.last_fetch_duration or 0)))
 
 
 async def wait_first(futures, instance):

nodeocc-1.0.27/src/view/transport.py

@@ -0,0 +1,46 @@
+"""Fragment large broadcast messages to fit in UDP datagrams."""
+import uuid
+import zlib
+
+PREFIX = b'nodeocc-fragment:'
+CHUNK_SIZE = 60000
+COMPRESSED = b'nodeocc-zlib:'
+
+
+def send(sock, port, data):
+    compressed = COMPRESSED + zlib.compress(data, level=1)
+    if len(compressed) < len(data):
+        data = compressed
+    if len(data) <= CHUNK_SIZE:
+        sock.sendto(data, ('<broadcast>', port))
+        return
+    message_id = uuid.uuid4().hex.encode('ascii')
+    count = (len(data) + CHUNK_SIZE - 1) // CHUNK_SIZE
+    for index in range(count):
+        header = PREFIX + message_id + b':' + str(index).encode() + b':' + str(count).encode() + b':'
+        sock.sendto(header + data[index * CHUNK_SIZE:(index + 1) * CHUNK_SIZE], ('<broadcast>', port))
+
+
+def receive(sock, max_buf):
+    fragments = {}
+    while True:
+        data, _ = sock.recvfrom(max_buf)
+        if not data.startswith(PREFIX):
+            return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data
+        try:
+            message_id, index, count, chunk = data[len(PREFIX):].split(b':', 3)
+            index, count = int(index), int(count)
+            if count < 2 or count > 10000 or not 0 <= index < count:
+                continue
+        except ValueError:
+            continue
+        # Retain only one message, so lost fragments cannot accumulate forever.
+        if message_id not in fragments:
+            fragments = {message_id: (count, {})}
+        expected, parts = fragments[message_id]
+        if count != expected:
+            continue
+        parts[index] = chunk
+        if len(parts) == count:
+            data = b''.join(parts[i] for i in range(count))
+            return zlib.decompress(data[len(COMPRESSED):]) if data.startswith(COMPRESSED) else data