nodeocc 1.0.23__tar.gz → 1.0.25__tar.gz

{nodeocc-1.0.23 → nodeocc-1.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: nodeocc
-Version: 1.0.23
+Version: 1.0.25
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.23 → nodeocc-1.0.25}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nodeocc"
-version = "1.0.23"
+version = "1.0.25"
 authors = [{ name = "Matteo Boschini" }]
 maintainers = [{ name = "Lorenzo Bonicelli" }, { name = "Aniello Panariello" }]
 description = "TUI application for viewing the status of GPU allocations on a Slurm cluster"

{nodeocc-1.0.23 → nodeocc-1.0.25}/src/controller/controller.py

@@ -1,3 +1,4 @@
+import random
 import sys
 import os
 from pathlib import Path
@@ -7,7 +8,7 @@ import numpy as np
 
 conf_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 print(conf_path)
-sys.path.append(conf_path)
+sys.path.insert(0, conf_path)
 from curses import wrapper
 from view.curses_multiwindow import main, Singleton, try_open_socket_as_slave
 
@@ -22,6 +23,7 @@ import time
 import json
 import argparse
 import traceback
+import socket
 from model.infrastructure import Infrastructure
 from model.job import Job
 import asyncio
@@ -64,13 +66,16 @@ BEGIN_DELIM = "!{$"
 END_DELIM_ENCODED = "!}$".encode('utf-8')
 EXTRA_MSG_BEGIN_DELIM = "!{#"
 MAX_BUF = 65535
-MAX_MSG_LEN = 1024 * 1024 * 1024
-JOB_LIMIT_PER_MSG = 60
+JOB_LIMIT_PER_MSG = 40
+TIMEOUT_MAX = 3
+n_timeout = 0
 
 parser = argparse.ArgumentParser(description='Visualize slurm jobs')
 parser.add_argument('--debug', action='store_true', help='Enable logging')
+parser.add_argument('--override', action='store_true', help='Enable logging')
 parser.add_argument('--master_only', action='store_true', help='Disable all prints - only run in background')
 parser.add_argument('--basepath', type=str, default='/tmp', help='Base path for nodeocc. Must be readable by all users')
+parser.add_argument('--diag', action='store_true', help='Run one-shot diagnostics and exit')
 args = parser.parse_args()
 
 # check basepath is readable
@@ -168,54 +173,77 @@ def decode_msg_slave(data, delim=BEGIN_DELIM):
 
 
 async def get_data_slave(instance):
+    global n_timeout
     inf, jobs, avg_wait_time = None, None, None
     try:
         # listen for data from master
         # instance.sock.settimeout(6.5)
         data, _ = instance.sock.recvfrom(MAX_BUF)
-        decoded_data = data.decode('utf-8')
-        if BEGIN_DELIM in decoded_data:
-            n_msgs_remaining = int(decoded_data.split(BEGIN_DELIM)[0])
-            # msg_len += len(str(msg_len)) message len is deprecated and used for n_msgs_remaining
-            instance.timeme(f"received first message of {n_msgs_remaining} (total len {len(data)} bytes)")
-
-            inf, jobs, avg_wait_time, decoded_msg = decode_msg_slave(data)
-            instance.timeme(f"- receive")
-            orig_timestamp = float(decoded_msg['ts'])
-
-            total_msg_n = n_msgs_remaining
-            msg_store = {i: [] for i in range(total_msg_n)}
-            while n_msgs_remaining > 0:
-                data, _ = instance.sock.recvfrom(MAX_BUF)
-                back_msg_idx = int(data.decode('utf-8').split(EXTRA_MSG_BEGIN_DELIM)[0])
-                msg_idx = total_msg_n - back_msg_idx - 1
-                _, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
-
-                # check if timestamp is the same
-                if float(new_ts['ts']) != orig_timestamp:
-                    instance.timeme(f"Timestamp mismatch, skipping")
-                    return inf, jobs, avg_wait_time  # return what we have
-
-                msg_store[msg_idx] += new_jobs
-
-                n_msgs_remaining -= 1
-                instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
-
-            # sort and merge
-            for msg_idx in range(total_msg_n):
-                if len(msg_store[msg_idx]) == 0:
-                    instance.log(f"Empty message {msg_idx}")
-                jobs += msg_store[msg_idx]
-        else:
-            instance.timeme(f"- no data")
-            return None, None, None, None
-
-    except BlockingIOError as e:
+        decoded_data = data.decode('utf-8', errors='replace')
+        if BEGIN_DELIM not in decoded_data:
+            if EXTRA_MSG_BEGIN_DELIM in decoded_data:
+                instance.log("Received extra packet before main packet, discarding")
+            else:
+                instance.log("Received packet without known delimiters, discarding")
+            return None, None, None
+
+        n_msgs_remaining = int(decoded_data.split(BEGIN_DELIM)[0])
+        # msg_len += len(str(msg_len)) message len is deprecated and used for n_msgs_remaining
+        instance.timeme(f"received first message of {n_msgs_remaining} (total len {len(data)} bytes)")
+
+        inf, jobs, avg_wait_time, decoded_msg = decode_msg_slave(data)
+        instance.timeme(f"- receive")
+        orig_timestamp = float(decoded_msg['ts'])
+
+        total_msg_n = n_msgs_remaining
+        msg_store = {i: [] for i in range(total_msg_n)}
+        while n_msgs_remaining > 0:
+            data, _ = instance.sock.recvfrom(MAX_BUF)
+            decoded_extra = data.decode('utf-8', errors='replace')
+            if EXTRA_MSG_BEGIN_DELIM not in decoded_extra:
+                instance.log("Received non-extra packet while waiting for extra packets, skipping")
+                continue
+
+            back_msg_idx = int(decoded_extra.split(EXTRA_MSG_BEGIN_DELIM)[0])
+            msg_idx = total_msg_n - back_msg_idx - 1
+            if msg_idx < 0 or msg_idx >= total_msg_n:
+                instance.log(f"Invalid extra packet index: {msg_idx}, skipping")
+                continue
+
+            _, new_jobs, _, new_ts = decode_msg_slave(data, delim=EXTRA_MSG_BEGIN_DELIM)
+
+            # check if timestamp is the same
+            if float(new_ts['ts']) != orig_timestamp:
+                instance.timeme(f"Timestamp mismatch, skipping")
+                return inf, jobs, avg_wait_time  # return what we have
+
+            msg_store[msg_idx] += new_jobs
+
+            n_msgs_remaining -= 1
+            instance.timeme(f"- extra receive, {n_msgs_remaining} remaining")
+
+        # sort and merge
+        for msg_idx in range(total_msg_n):
+            if len(msg_store[msg_idx]) == 0:
+                instance.log(f"Empty message {msg_idx}")
+            jobs += msg_store[msg_idx]
+
+        n_timeout = 0
+
+    except BlockingIOError:
         pass
 
-    except TimeoutError as e:
-        instance.log(f"TIMEOUT")
-        try_open_socket_as_slave(instance)
+    except TimeoutError:
+        n_timeout += 1
+        instance.log(f"TIMEOUT {n_timeout}/{TIMEOUT_MAX}")
+        if n_timeout > TIMEOUT_MAX:
+            try_open_socket_as_slave(instance, force=True)
+
+        time.sleep(random.randint(100, 10000) / 1000)
+
+    except Exception as e:
+        instance.err(f"Exception while receiving data from master: {e}")
+        instance.err(traceback.format_exc())
 
     return inf, jobs, avg_wait_time
 
@@ -264,7 +292,53 @@ def display_main(stdscr):
     return asyncio.run(main(stdscr))
 
 
+def run_diagnostics(instance: Singleton):
+    hostname = socket.getfqdn().split('.')[0]
+    print(f"Diagnostics for nodeocc on host: {hostname}")
+    print(f"Base path: {instance.basepath}")
+    print(f"Role detected: {'master' if instance.is_master else 'slave'}")
+    print("")
+
+    try:
+        port_files = instance.get_port_files()
+        print(f"Visible port files: {len(port_files)}")
+        for pf in port_files:
+            try:
+                pid = Path(pf).read_text().strip()
+            except Exception:
+                pid = "?"
+            print(f"  - {pf} (pid={pid})")
+    except Exception as e:
+        print(f"Could not list port files: {e}")
+
+    print("")
+
+    if instance.is_master:
+        print("Running one master update cycle...")
+        try:
+            inf, jobs = update_data_master(instance)
+            print(f"Master update OK. Nodes={len(inf.nodes)}, Jobs={len(jobs)}, Port={instance.port}")
+        except Exception as e:
+            print(f"Master update FAILED: {e}")
+            print(traceback.format_exc())
+    else:
+        print("Waiting for one packet from master...")
+        try:
+            inf, jobs, _ = asyncio.run(get_data_slave(instance))
+            if inf is None or jobs is None:
+                print("No valid payload received from master.")
+            else:
+                print(f"Slave receive OK. Jobs in snapshot={len(jobs)}")
+        except Exception as e:
+            print(f"Slave receive FAILED: {e}")
+            print(traceback.format_exc())
+
+
 def _main():
+    if args.diag:
+        run_diagnostics(orig_instance)
+        return
+
     if args.master_only:
         assert orig_instance.is_master, "Instance is not `master`, run with `force_override` to override"
 

{nodeocc-1.0.23 → nodeocc-1.0.25}/src/nodeocc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: nodeocc
-Version: 1.0.23
+Version: 1.0.25
 Summary: TUI application for viewing the status of GPU allocations on a Slurm cluster
 Author: Matteo Boschini
 Maintainer: Lorenzo Bonicelli, Aniello Panariello

{nodeocc-1.0.23 → nodeocc-1.0.25}/src/readers/slurmreader.py

@@ -45,24 +45,54 @@ def _node_from_sinfo(x):
     return n
 
 
-def _parse_gpu_models(line):
-    line = line[0].split(',')
-    if line[0] == '(null)':
+def _parse_gpu_count(gres):
+    if not isinstance(gres, str):
+        return 0
+    gres = gres.strip()
+    if gres in ('(null)', ''):
+        return 0
+
+    n_gpu = 0
+    for token in gres.split(','):
+        token = token.strip()
+        if not token.lower().startswith('gpu'):
+            continue
+        last_field = token.split(':')[-1].split('(')[0]
+        match = re.search(r'\d+', last_field)
+        n_gpu += int(match.group(0)) if match else 1
+    return n_gpu
+
+
+def _parse_gpu_models(gres):
+    if not isinstance(gres, str):
         return 'n/a'
-    else:
-        if len(line) == 1:
-            return line[0].split(':')[1]
+    gres = gres.strip()
+    if gres in ('(null)', ''):
+        return 'n/a'
+
+    models = []
+    for token in gres.split(','):
+        token = token.strip()
+        if not token.lower().startswith('gpu'):
+            continue
+        parts = token.split(':')
+        if len(parts) >= 3:
+            model = parts[1]
+        elif len(parts) == 2:
+            model = 'gpu'
         else:
-            # return line[0].split(':')[1] + ',' + line[1].split(':')[1]
-            return ','.join([x.split(':')[1] for x in line])
+            model = 'n/a'
+        if model not in models:
+            models.append(model)
+
+    return ','.join(models) if models else 'n/a'
 
 
 def _read_nodes(reservations=None, pending_res=None, return_raw=False):
     sinfo_df = pd.read_csv(StringIO(os.popen(r'sinfo -N -o "%N;%G;%t;%m;%c;%f"').read()), sep=';')
     sinfo_df = _split_column(sinfo_df, 'NODELIST').drop_duplicates()
-    sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(lambda x: 0 if x.split('(')[0].split(':')[0] in ('(null)', '') else int(x.split('(')[0].split(':')[-1]))
-    # sinfo_df['m_gpu'] = sinfo_df['GRES'].str.split(':').apply(lambda x: 'n/a' if x[0] in ('(null)', '') else x[1])
-    sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].str.split(';').apply(_parse_gpu_models)
+    sinfo_df.loc[:, 'n_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_count)
+    sinfo_df.loc[:, 'm_gpu'] = sinfo_df['GRES'].apply(_parse_gpu_models)
     sinfo_df['reserved'] = 'no'
     if reservations is not None:
         sinfo_df.loc[sinfo_df['NODELIST'].isin(reservations.Nodes.unique()), 'reserved'] = 'yes'

{nodeocc-1.0.23 → nodeocc-1.0.25}/src/view/__init__.py

@@ -3,6 +3,8 @@ from . import slurm_list, slurm_viz, styles
 
 def update_views(stdscr, instance, filter):
     if instance.jobs is None or instance.inf is None:
+        instance.rens = "No data received from master yet.\nTry running with --debug or --diag."
+        instance.nocc = "Waiting for infrastructure snapshot..."
         return
 
     instance.rens = slurm_list.view_list(

{nodeocc-1.0.23 → nodeocc-1.0.25}/src/view/curses_multiwindow.py

@@ -44,12 +44,49 @@ def is_file_readable_byall(file_path):
     return os.stat(file_path).st_mode & stat.S_IROTH
 
 
-def try_open_socket_as_slave(instance):
+def is_nodeocc_process(pid):
+    try:
+        cmdline = ' '.join(psutil.Process(int(pid)).cmdline())
+        return ('nodeocc' in cmdline) or ('controller.py' in cmdline)
+    except Exception:
+        return False
+
+
+def port_from_portfile(portfile_path):
+    try:
+        return int(Path(portfile_path).name.split('.')[0].split('master_')[1])
+    except Exception:
+        return None
+
+
+def process_has_udp_port(pid, port):
+    if port is None:
+        return False
+    try:
+        proc = psutil.Process(int(pid))
+        for conn in proc.connections(kind='udp'):
+            if conn.laddr and conn.laddr.port == int(port):
+                return True
+    except Exception:
+        return False
+    return False
+
+
+def try_open_socket_as_slave(instance, force=False):
     if len(instance.get_port_files()) == 0:
         raise Exception("No master running")
 
-    cur_port = [int(f.split('.')[0].split('master_')[1]) for f in instance.get_port_files()
-                if (pid := Path(f).read_text()) and psutil.pid_exists(int(pid))][0]
+    active_ports = []
+    for f in instance.get_port_files():
+        pid = Path(f).read_text().strip()
+        port = port_from_portfile(f)
+        if pid and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
+            active_ports.append(port)
+
+    if len(active_ports) == 0:
+        raise Exception("No active master process found from port files")
+
+    cur_port = active_ports[0]
 
     if hasattr(instance, 'port'):
         # check if port file has same name
@@ -57,7 +94,7 @@ def try_open_socket_as_slave(instance):
             instance.log(f"- MASTER HAS CHANGED PORT: {instance.port} vs {cur_port}")
 
     instance.port = cur_port
-    instance._open_socket_as_slave(instance.port)
+    instance._open_socket_as_slave(instance.port, force=force)
 
     if instance.try_open_counter > 5:
         instance.err(f"Could not open socket as slave on port {instance.port}")
@@ -90,21 +127,26 @@ class Singleton:
             logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S',
                                 handlers=[handler])
 
-    def clean_port_files(self):
+    def clean_port_files(self, force=False):
         nodename = socket.getfqdn().split('.')[0]
         bp = os.path.join(self.basepath, nodename)
         if not os.path.exists(bp):
             return True
         for f in os.listdir(bp):
-            if 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(os.path.join(bp, f)):
-                # read pid from file
-                pid = Path(os.path.join(bp, f)).read_text()
-                # assert no process is running
-                if psutil.pid_exists(int(pid)):
-                    self.log(f"Process {pid} is still running, killing self")
+            fp = os.path.join(bp, f)
+            if 'master_' in f and f.endswith('.portfile') and is_file_writable_byall(fp):
+                pid = Path(fp).read_text().strip()
+                port = port_from_portfile(fp)
+                if not pid:
+                    os.remove(fp)
+                    continue
+                if pid == str(os.getpid()):
+                    os.remove(fp)
+                    continue
+                if (not force) and is_nodeocc_process(pid) and process_has_udp_port(pid, port):
+                    self.log(f"Process {pid} is still running, keeping existing master")
                     return False
-
-                os.system(f"rm {os.path.join(bp, f)}")
+                os.remove(fp)
         return True
 
     def __init__(self, args=None):
@@ -115,6 +157,7 @@ class Singleton:
             Singleton.__instance = self
 
         self.args = args
+        self.basepath = self.args.basepath
         self.setup_logging()
 
         self.try_open_counter = 0
@@ -141,15 +184,17 @@ class Singleton:
         self.k = -1
 
         # check if user group is 'student' or 'tesisti'
-        user_group = grp.getgrgid(os.getgid()).gr_name
+        try:
+            user_group = grp.getgrgid(os.getgid()).gr_name
+        except KeyError:
+            user_group = ''
+            self.err(f"Could not resolve gid {os.getgid()} via grp.getgrgid")
         self.cur_partition = 'stud' if user_group in ['studenti', 'tesisti'] else 'prod'
 
-        self.basepath = self.args.basepath
-
         self.is_master = not self.check_existing_master_running()
 
-        if self.is_master:
-            self.port = self.create_socket_as_master()
+        if self.is_master or args.override:
+            self.port = self.create_socket_as_master(force=args.override)
         else:
             try_open_socket_as_slave(self)
 
@@ -158,9 +203,16 @@ class Singleton:
         if len(portfiles) > 0:
             for f in portfiles:
                 # check if process is still running
-                pid = Path(f).read_text()
-                if pid and int(pid) != os.getpid():
-                    return psutil.pid_exists(int(pid))
+                pid = Path(f).read_text().strip()
+                if not pid:
+                    continue
+                try:
+                    pid_int = int(pid)
+                except ValueError:
+                    continue
+                port = port_from_portfile(f)
+                if pid_int != os.getpid() and is_nodeocc_process(pid_int) and process_has_udp_port(pid_int, port):
+                    return True
         return False
 
     def get_port_files(self):
@@ -176,31 +228,12 @@ class Singleton:
         portfiles = [f for f in portfiles if is_file_readable_byall(f) and is_file_writable_byall(f)]
         return portfiles
 
-    def create_socket_as_master(self):
+    def update_port_file(self, force=False):
         nodename = socket.getfqdn().split('.')[0]
 
-        # create udp socket for broadcasting
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
-
-        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-
-        # Enable broadcasting mode
-        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
-
-        self.sock.bind(('', 0))
-
-        self.port = self.sock.getsockname()[1]
-        self.sock.settimeout(2)
-        self.log(f"Socket created as master on port {self.port}")
-
         # get pid of current process
         self.pid = os.getpid()
 
-        # clean up old port files
-        kill_ok = self.clean_port_files()
-        if not kill_ok:
-            return None
-
         try:
             # create file to store port with 666 permissions to file
             # file is in folder basepath/nodename/master_{port}.portfile
@@ -225,12 +258,34 @@ class Singleton:
             # check if file exists
             if filepath.exists():
                 # delete file
-                os.system(f"rm {filepath}")
+                os.remove(filepath)
             return None
 
+    def create_socket_as_master(self, force=False):
+        # clean up old port files before creating the socket to avoid inconsistent state
+        kill_ok = self.clean_port_files(force=force)
+        if not kill_ok:
+            return None
+
+        # create udp socket for broadcasting
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
+
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        # Enable broadcasting mode
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+
+        self.sock.bind(('', 0))
+
+        self.port = self.sock.getsockname()[1]
+        self.sock.settimeout(2)
+        self.log(f"Socket created as master on port {self.port}")
+
+        self.update_port_file(force=force)
+
         return self.port
 
-    def _open_socket_as_slave(self, port):
+    def _open_socket_as_slave(self, port, force=False):
         try:
             # create udp socket for broadcasting
             self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)